Setup

library(tidyverse)
library(janitor)
library(tools)
library(caret)
library(ROCR)
library(caret)
compute_auroc <- function(predictions, labels, title) {
    pred <- prediction(predictions, labels)
    auroc <- performance(pred, measure = 'auc')
    auroc <- auroc@y.values[[1]]
    perf <- performance(pred, 'tpr', 'fpr')
    plot(perf,
         main = title,
         xlim=c(0,1),
         ylim=c(0,1),
         colorize = FALSE)
    abline(a = 0,
           b = 1,
           col = 'red',
           lty = 2)
    text(0.8,
         0.25,
         paste('AUROC:', sprintf('%.3f', round(auroc, 3))))
}
compute_aupr <- function(predictions, labels, title) {
    n_black <- length(labels[labels == 1])
    n_total <- length(labels)
    pred <- prediction(predictions, labels)
    aupr <- performance(pred, measure = 'aucpr')
    aupr <- aupr@y.values[[1]]
    perf <- performance(pred, 'prec', 'rec')
    plot(perf,
         main = title,
         xlim=c(0,1),
         ylim=c(0,1),
         colorize = FALSE)
    abline(a = n_black/n_total,
           b = 0,
           col = 'red',
           lty = 2)
    text(0.2,
         0.2,
         paste('Null AUPR:', sprintf('%.3f', round(n_black / n_total, 3))))
    text(0.8,
         0.2,
         paste('AUPR:', sprintf('%.3f', round(aupr, 3))))
}
confusion_matrix <- function(df, column, threshold=0.5) {
    preds <- as.factor(if_else(df[column] >= threshold, 1, 0))
    confusionMatrix(preds, df$label, positive = '1')
}

Load data

irop_data <- read_csv('/Volumes/External/irop_data/irop_07092020.csv') %>%
    clean_names() %>%
    distinct(subject_id, .keep_all = TRUE) %>%
    select(subject_id, race)

── Column specification ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  .default = col_character(),
  subjectVisitID = col_double(),
  birthWeight = col_double(),
  gestationalAgeWeeks = col_double(),
  gestationalAgeDays = col_double(),
  `PMA Weeks` = col_double(),
  `PMA Days` = col_double(),
  PMARaw = col_double(),
  `Session Followup` = col_double(),
  `Golden Reading Stage` = col_double(),
  bloodTaken = col_double(),
  salivaTaken = col_double()
)
ℹ Use `spec()` for the full column specifications.
Warning: 5380 parsing failures.
 row              col expected actual                                            file
1645 subjectVisitID   a double   NULL '/Volumes/External/irop_data/irop_07092020.csv'
1645 PMA Weeks        a double   NULL '/Volumes/External/irop_data/irop_07092020.csv'
1645 PMA Days         a double   NULL '/Volumes/External/irop_data/irop_07092020.csv'
1645 PMARaw           a double   NULL '/Volumes/External/irop_data/irop_07092020.csv'
1645 Session Followup a double   NULL '/Volumes/External/irop_data/irop_07092020.csv'
.... ................ ........ ...... ...............................................
See problems(...) for more details.
test_data <- read_csv('./out/datasets/test_data.csv') %>%
    select(subject_id, image_id)

── Column specification ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  subject_id = col_character(),
  race = col_character(),
  variable = col_character(),
  value = col_character(),
  image_id = col_double(),
  fundus_location = col_character(),
  segmentation_location = col_character()
)
image_level <- read_csv('./out/probabilities/retcam_filtered_0.csv', col_types = cols()) %>%
    mutate(label = as.factor(if_else(str_detect(img_loc, 'black'), 1, 0)),
           image_id = as.numeric(file_path_sans_ext(basename(img_loc)))) %>%
    select(img_loc, image_id, label, retcam = probability) %>%
    bind_cols(select(read_csv('./out/probabilities/retcam_filtered_0_random.csv', col_types = cols()), retcam_random = probability)) %>%
    
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_random.csv', col_types = cols()), filter_0_random = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0.csv', col_types = cols()), filter_0 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_binarized.csv', col_types = cols()), filter_0_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_skeletonized.csv', col_types = cols()), filter_0_skeletonized = probability)) %>%
    
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_50.csv', col_types = cols()), filter_50 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_50_binarized.csv', col_types = cols()), filter_50_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_50_skeletonized.csv', col_types = cols()), filter_50_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_100.csv', col_types = cols()), filter_100 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_100_binarized.csv', col_types = cols()), filter_100_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_100_skeletonized.csv', col_types = cols()), filter_100_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_150.csv', col_types = cols()), filter_150 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_150_binarized.csv', col_types = cols()), filter_150_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_150_skeletonized.csv', col_types = cols()), filter_150_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_200.csv', col_types = cols()), filter_200 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_200_binarized.csv', col_types = cols()), filter_200_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_200_skeletonized.csv', col_types = cols()), filter_200_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_210.csv', col_types = cols()), filter_210 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_210_binarized.csv', col_types = cols()), filter_210_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_210_skeletonized.csv', col_types = cols()), filter_210_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_220.csv', col_types = cols()), filter_220 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_220_binarized.csv', col_types = cols()), filter_220_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_220_skeletonized.csv', col_types = cols()), filter_220_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_230.csv', col_types = cols()), filter_230 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_230_binarized.csv', col_types = cols()), filter_230_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_230_skeletonized.csv', col_types = cols()), filter_230_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_240.csv', col_types = cols()), filter_240 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_240_binarized.csv', col_types = cols()), filter_240_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_240_skeletonized.csv', col_types = cols()), filter_240_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_250.csv', col_types = cols()), filter_250 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_250_binarized.csv', col_types = cols()), filter_250_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_250_skeletonized.csv', col_types = cols()), filter_250_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_257.csv', col_types = cols()), filter_257 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_257_binarized.csv', col_types = cols()), filter_257_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_257_skeletonized.csv', col_types = cols()), filter_257_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_10.csv', col_types = cols()), filter_10 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_10_binarized.csv', col_types = cols()), filter_10_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_0_10_skeletonized.csv', col_types = cols()), filter_10_skeletonized = probability)) %>%

    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_75_150.csv', col_types = cols()), filter_75 = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_75_150_binarized.csv', col_types = cols()), filter_75_binarized = probability)) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_filtered_75_150_skeletonized.csv', col_types = cols()), filter_75_skeletonized = probability)) %>%
    
    inner_join(test_data, by = 'image_id') %>%
    select(subject_id, everything(), -img_loc)



subject_level <- image_level %>%
    group_by(subject_id) %>%
    mutate(across(c(-image_id, -label), median)) %>%
    ungroup() %>%
    distinct(subject_id, .keep_all = TRUE)

Image-level Analysis

RetCam Images

compute_aupr(image_level$retcam, image_level$label, 'PR: Raw RetCam Images')

compute_auroc(image_level$retcam, image_level$label, 'ROC: Raw RetCam Images')

confusion_matrix(image_level, 'retcam')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 819  10
         1   8 513
                                         
               Accuracy : 0.9867         
                 95% CI : (0.979, 0.9921)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : <2e-16         
                                         
                  Kappa : 0.9719         
                                         
 Mcnemar's Test P-Value : 0.8137         
                                         
            Sensitivity : 0.9809         
            Specificity : 0.9903         
         Pos Pred Value : 0.9846         
         Neg Pred Value : 0.9879         
             Prevalence : 0.3874         
         Detection Rate : 0.3800         
   Detection Prevalence : 0.3859         
      Balanced Accuracy : 0.9856         
                                         
       'Positive' Class : 1              
                                         
compute_aupr(image_level$retcam_random, image_level$label, 'PR: Raw RetCam Images - Shuffled Labels')

compute_auroc(image_level$retcam_random, image_level$label, 'ROC: Raw RetCam Images - Shuffled Labels')

confusion_matrix(image_level, 'retcam_random')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 444 113
         1 383 410
                                          
               Accuracy : 0.6326          
                 95% CI : (0.6062, 0.6584)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 0.06902         
                                          
                  Kappa : 0.293           
                                          
 Mcnemar's Test P-Value : < 2e-16         
                                          
            Sensitivity : 0.7839          
            Specificity : 0.5369          
         Pos Pred Value : 0.5170          
         Neg Pred Value : 0.7971          
             Prevalence : 0.3874          
         Detection Rate : 0.3037          
   Detection Prevalence : 0.5874          
      Balanced Accuracy : 0.6604          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 0

compute_aupr(image_level$filter_0, image_level$label, 'PR: Segmentations: zero all pixels < 0')

compute_auroc(image_level$filter_0, image_level$label, 'ROC: Segmentations: zero all pixels < 0')

confusion_matrix(image_level, 'filter_0')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 758  74
         1  69 449
                                        
               Accuracy : 0.8941        
                 95% CI : (0.8764, 0.91)
    No Information Rate : 0.6126        
    P-Value [Acc > NIR] : <2e-16        
                                        
                  Kappa : 0.7764        
                                        
 Mcnemar's Test P-Value : 0.738         
                                        
            Sensitivity : 0.8585        
            Specificity : 0.9166        
         Pos Pred Value : 0.8668        
         Neg Pred Value : 0.9111        
             Prevalence : 0.3874        
         Detection Rate : 0.3326        
   Detection Prevalence : 0.3837        
      Balanced Accuracy : 0.8875        
                                        
       'Positive' Class : 1             
                                        
compute_aupr(image_level$filter_0_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 0 and binarize')

compute_auroc(image_level$filter_0_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 0 and binarize')

confusion_matrix(image_level, 'filter_0_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 733  31
         1  94 492
                                          
               Accuracy : 0.9074          
                 95% CI : (0.8907, 0.9223)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.8091          
                                          
 Mcnemar's Test P-Value : 2.932e-08       
                                          
            Sensitivity : 0.9407          
            Specificity : 0.8863          
         Pos Pred Value : 0.8396          
         Neg Pred Value : 0.9594          
             Prevalence : 0.3874          
         Detection Rate : 0.3644          
   Detection Prevalence : 0.4341          
      Balanced Accuracy : 0.9135          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_0_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 0 and skeletonize')

compute_auroc(image_level$filter_0_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 0 and skeletonize')

confusion_matrix(image_level, 'filter_0_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 691  38
         1 136 485
                                          
               Accuracy : 0.8711          
                 95% CI : (0.8521, 0.8885)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.7375          
                                          
 Mcnemar's Test P-Value : 1.93e-13        
                                          
            Sensitivity : 0.9273          
            Specificity : 0.8356          
         Pos Pred Value : 0.7810          
         Neg Pred Value : 0.9479          
             Prevalence : 0.3874          
         Detection Rate : 0.3593          
   Detection Prevalence : 0.4600          
      Balanced Accuracy : 0.8814          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_0_random, image_level$label, 'PR: Segmentations: zero all pixels < 0 - Shuffled Labels')

compute_auroc(image_level$filter_0_random, image_level$label, 'ROC: Segmentations: zero all pixels < 0 - Shuffled Labels')

confusion_matrix(image_level, 'filter_0_random')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 703 418
         1 124 105
                                          
               Accuracy : 0.5985          
                 95% CI : (0.5718, 0.6248)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 0.8619          
                                          
                  Kappa : 0.0567          
                                          
 Mcnemar's Test P-Value : <2e-16          
                                          
            Sensitivity : 0.20076         
            Specificity : 0.85006         
         Pos Pred Value : 0.45852         
         Neg Pred Value : 0.62712         
             Prevalence : 0.38741         
         Detection Rate : 0.07778         
   Detection Prevalence : 0.16963         
      Balanced Accuracy : 0.52541         
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 50

compute_aupr(image_level$filter_50, image_level$label, 'PR: Segmentations: zero all pixels < 50')

compute_auroc(image_level$filter_50, image_level$label, 'ROC: Segmentations: zero all pixels < 50')

confusion_matrix(image_level, 'filter_50')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 588  63
         1 239 460
                                          
               Accuracy : 0.7763          
                 95% CI : (0.7531, 0.7983)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.5561          
                                          
 Mcnemar's Test P-Value : < 2.2e-16       
                                          
            Sensitivity : 0.8795          
            Specificity : 0.7110          
         Pos Pred Value : 0.6581          
         Neg Pred Value : 0.9032          
             Prevalence : 0.3874          
         Detection Rate : 0.3407          
   Detection Prevalence : 0.5178          
      Balanced Accuracy : 0.7953          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_50_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 50 and binarize')

compute_auroc(image_level$filter_50_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 50 and binarize')

confusion_matrix(image_level, 'filter_0_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 733  31
         1  94 492
                                          
               Accuracy : 0.9074          
                 95% CI : (0.8907, 0.9223)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.8091          
                                          
 Mcnemar's Test P-Value : 2.932e-08       
                                          
            Sensitivity : 0.9407          
            Specificity : 0.8863          
         Pos Pred Value : 0.8396          
         Neg Pred Value : 0.9594          
             Prevalence : 0.3874          
         Detection Rate : 0.3644          
   Detection Prevalence : 0.4341          
      Balanced Accuracy : 0.9135          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_50_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 50 and skeletonize')

compute_auroc(image_level$filter_50_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 50 and skeletonize')

confusion_matrix(image_level, 'filter_50_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 408  20
         1 419 503
                                          
               Accuracy : 0.6748          
                 95% CI : (0.6491, 0.6998)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 1.202e-06       
                                          
                  Kappa : 0.3991          
                                          
 Mcnemar's Test P-Value : < 2.2e-16       
                                          
            Sensitivity : 0.9618          
            Specificity : 0.4933          
         Pos Pred Value : 0.5456          
         Neg Pred Value : 0.9533          
             Prevalence : 0.3874          
         Detection Rate : 0.3726          
   Detection Prevalence : 0.6830          
      Balanced Accuracy : 0.7276          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 100

compute_aupr(image_level$filter_100, image_level$label, 'PR: Segmentations: zero all pixels < 100')

compute_auroc(image_level$filter_100, image_level$label, 'ROC: Segmentations: zero all pixels < 100')

confusion_matrix(image_level, 'filter_100')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 736 210
         1  91 313
                                         
               Accuracy : 0.777          
                 95% CI : (0.7539, 0.799)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : < 2.2e-16      
                                         
                  Kappa : 0.5098         
                                         
 Mcnemar's Test P-Value : 1.036e-11      
                                         
            Sensitivity : 0.5985         
            Specificity : 0.8900         
         Pos Pred Value : 0.7748         
         Neg Pred Value : 0.7780         
             Prevalence : 0.3874         
         Detection Rate : 0.2319         
   Detection Prevalence : 0.2993         
      Balanced Accuracy : 0.7442         
                                         
       'Positive' Class : 1              
                                         
compute_aupr(image_level$filter_100_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 100 and binarize')

compute_auroc(image_level$filter_100_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 100 and binarize')

confusion_matrix(image_level, 'filter_100_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 669 142
         1 158 381
                                          
               Accuracy : 0.7778          
                 95% CI : (0.7546, 0.7997)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : <2e-16          
                                          
                  Kappa : 0.5344          
                                          
 Mcnemar's Test P-Value : 0.3865          
                                          
            Sensitivity : 0.7285          
            Specificity : 0.8089          
         Pos Pred Value : 0.7069          
         Neg Pred Value : 0.8249          
             Prevalence : 0.3874          
         Detection Rate : 0.2822          
   Detection Prevalence : 0.3993          
      Balanced Accuracy : 0.7687          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_100_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 100 and skeletonize')

compute_auroc(image_level$filter_100_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 100 and skeletonize')

confusion_matrix(image_level, 'filter_100_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 632 144
         1 195 379
                                          
               Accuracy : 0.7489          
                 95% CI : (0.7249, 0.7718)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.4803          
                                          
 Mcnemar's Test P-Value : 0.006615        
                                          
            Sensitivity : 0.7247          
            Specificity : 0.7642          
         Pos Pred Value : 0.6603          
         Neg Pred Value : 0.8144          
             Prevalence : 0.3874          
         Detection Rate : 0.2807          
   Detection Prevalence : 0.4252          
      Balanced Accuracy : 0.7444          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 150

compute_aupr(image_level$filter_150, image_level$label, 'PR: Segmentations: zero all pixels < 150')

compute_auroc(image_level$filter_150, image_level$label, 'ROC: Segmentations: zero all pixels < 150')

confusion_matrix(image_level, 'filter_150')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 625 118
         1 202 405
                                          
               Accuracy : 0.763           
                 95% CI : (0.7393, 0.7854)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.5149          
                                          
 Mcnemar's Test P-Value : 3.487e-06       
                                          
            Sensitivity : 0.7744          
            Specificity : 0.7557          
         Pos Pred Value : 0.6672          
         Neg Pred Value : 0.8412          
             Prevalence : 0.3874          
         Detection Rate : 0.3000          
   Detection Prevalence : 0.4496          
      Balanced Accuracy : 0.7651          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_150_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 150 and binarize')

compute_auroc(image_level$filter_150_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 150 and binarize')

confusion_matrix(image_level, 'filter_150_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 702 167
         1 125 356
                                          
               Accuracy : 0.7837          
                 95% CI : (0.7608, 0.8054)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2e-16         
                                          
                  Kappa : 0.5375          
                                          
 Mcnemar's Test P-Value : 0.01642         
                                          
            Sensitivity : 0.6807          
            Specificity : 0.8489          
         Pos Pred Value : 0.7401          
         Neg Pred Value : 0.8078          
             Prevalence : 0.3874          
         Detection Rate : 0.2637          
   Detection Prevalence : 0.3563          
      Balanced Accuracy : 0.7648          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_150_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 150 and skeletonize')

compute_auroc(image_level$filter_150_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 150 and skeletonize')

confusion_matrix(image_level, 'filter_150_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 641 162
         1 186 361
                                         
               Accuracy : 0.7422         
                 95% CI : (0.718, 0.7654)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : <2e-16         
                                         
                  Kappa : 0.4614         
                                         
 Mcnemar's Test P-Value : 0.2176         
                                         
            Sensitivity : 0.6902         
            Specificity : 0.7751         
         Pos Pred Value : 0.6600         
         Neg Pred Value : 0.7983         
             Prevalence : 0.3874         
         Detection Rate : 0.2674         
   Detection Prevalence : 0.4052         
      Balanced Accuracy : 0.7327         
                                         
       'Positive' Class : 1              
                                         

Segmentations: zero all pixels < 200

compute_aupr(image_level$filter_200, image_level$label, 'PR: Segmentations: zero all pixels < 200')

compute_auroc(image_level$filter_200, image_level$label, 'ROC: Segmentations: zero all pixels < 200')

confusion_matrix(image_level, 'filter_200')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 548 121
         1 279 402
                                         
               Accuracy : 0.7037         
                 95% CI : (0.6786, 0.728)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : 1.64e-12       
                                         
                  Kappa : 0.4086         
                                         
 Mcnemar's Test P-Value : 4.16e-15       
                                         
            Sensitivity : 0.7686         
            Specificity : 0.6626         
         Pos Pred Value : 0.5903         
         Neg Pred Value : 0.8191         
             Prevalence : 0.3874         
         Detection Rate : 0.2978         
   Detection Prevalence : 0.5044         
      Balanced Accuracy : 0.7156         
                                         
       'Positive' Class : 1              
                                         
compute_aupr(image_level$filter_200_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 200 and binarize')

compute_auroc(image_level$filter_200_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 200 and binarize')

confusion_matrix(image_level, 'filter_200_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 661 233
         1 166 290
                                          
               Accuracy : 0.7044          
                 95% CI : (0.6793, 0.7287)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 1.083e-12       
                                          
                  Kappa : 0.3623          
                                          
 Mcnemar's Test P-Value : 0.0009527       
                                          
            Sensitivity : 0.5545          
            Specificity : 0.7993          
         Pos Pred Value : 0.6360          
         Neg Pred Value : 0.7394          
             Prevalence : 0.3874          
         Detection Rate : 0.2148          
   Detection Prevalence : 0.3378          
      Balanced Accuracy : 0.6769          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_200_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 200 and skeletonize')

compute_auroc(image_level$filter_200_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 200 and skeletonize')

confusion_matrix(image_level, 'filter_200_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 766 365
         1  61 158
                                          
               Accuracy : 0.6844          
                 95% CI : (0.6589, 0.7092)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 2.354e-08       
                                          
                  Kappa : 0.2557          
                                          
 Mcnemar's Test P-Value : < 2.2e-16       
                                          
            Sensitivity : 0.3021          
            Specificity : 0.9262          
         Pos Pred Value : 0.7215          
         Neg Pred Value : 0.6773          
             Prevalence : 0.3874          
         Detection Rate : 0.1170          
   Detection Prevalence : 0.1622          
      Balanced Accuracy : 0.6142          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 210

compute_aupr(image_level$filter_210, image_level$label, 'PR: Segmentations: zero all pixels < 210')

compute_auroc(image_level$filter_210, image_level$label, 'ROC: Segmentations: zero all pixels < 210')

confusion_matrix(image_level, 'filter_210')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 712 326
         1 115 197
                                          
               Accuracy : 0.6733          
                 95% CI : (0.6476, 0.6983)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 2.096e-06       
                                          
                  Kappa : 0.2566          
                                          
 Mcnemar's Test P-Value : < 2.2e-16       
                                          
            Sensitivity : 0.3767          
            Specificity : 0.8609          
         Pos Pred Value : 0.6314          
         Neg Pred Value : 0.6859          
             Prevalence : 0.3874          
         Detection Rate : 0.1459          
   Detection Prevalence : 0.2311          
      Balanced Accuracy : 0.6188          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_210_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 210 and binarize')

compute_auroc(image_level$filter_210_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 210 and binarize')

confusion_matrix(image_level, 'filter_210_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 639 203
         1 188 320
                                          
               Accuracy : 0.7104          
                 95% CI : (0.6854, 0.7345)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 3.463e-14       
                                          
                  Kappa : 0.3866          
                                          
 Mcnemar's Test P-Value : 0.4789          
                                          
            Sensitivity : 0.6119          
            Specificity : 0.7727          
         Pos Pred Value : 0.6299          
         Neg Pred Value : 0.7589          
             Prevalence : 0.3874          
         Detection Rate : 0.2370          
   Detection Prevalence : 0.3763          
      Balanced Accuracy : 0.6923          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_210_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 210 and skeletonize')

compute_auroc(image_level$filter_210_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 210 and skeletonize')

confusion_matrix(image_level, 'filter_210_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 562 160
         1 265 363
                                          
               Accuracy : 0.6852          
                 95% CI : (0.6597, 0.7099)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 1.699e-08       
                                          
                  Kappa : 0.3603          
                                          
 Mcnemar's Test P-Value : 4.541e-07       
                                          
            Sensitivity : 0.6941          
            Specificity : 0.6796          
         Pos Pred Value : 0.5780          
         Neg Pred Value : 0.7784          
             Prevalence : 0.3874          
         Detection Rate : 0.2689          
   Detection Prevalence : 0.4652          
      Balanced Accuracy : 0.6868          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 220

compute_aupr(image_level$filter_220, image_level$label, 'PR: Segmentations: zero all pixels < 220')

compute_auroc(image_level$filter_220, image_level$label, 'ROC: Segmentations: zero all pixels < 220')

confusion_matrix(image_level, 'filter_220')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 576 224
         1 251 299
                                         
               Accuracy : 0.6481         
                 95% CI : (0.622, 0.6736)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : 0.003819       
                                         
                  Kappa : 0.2657         
                                         
 Mcnemar's Test P-Value : 0.232884       
                                         
            Sensitivity : 0.5717         
            Specificity : 0.6965         
         Pos Pred Value : 0.5436         
         Neg Pred Value : 0.7200         
             Prevalence : 0.3874         
         Detection Rate : 0.2215         
   Detection Prevalence : 0.4074         
      Balanced Accuracy : 0.6341         
                                         
       'Positive' Class : 1              
                                         
compute_aupr(image_level$filter_220_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 220 and binarize')

compute_auroc(image_level$filter_220_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 220 and binarize')

confusion_matrix(image_level, 'filter_220_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 545 162
         1 282 361
                                          
               Accuracy : 0.6711          
                 95% CI : (0.6453, 0.6961)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 4.710e-06       
                                          
                  Kappa : 0.3351          
                                          
 Mcnemar's Test P-Value : 1.628e-08       
                                          
            Sensitivity : 0.6902          
            Specificity : 0.6590          
         Pos Pred Value : 0.5614          
         Neg Pred Value : 0.7709          
             Prevalence : 0.3874          
         Detection Rate : 0.2674          
   Detection Prevalence : 0.4763          
      Balanced Accuracy : 0.6746          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_220_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 220 and skeletonize')

compute_auroc(image_level$filter_220_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 220 and skeletonize')

confusion_matrix(image_level, 'filter_220_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 800 450
         1  27  73
                                          
               Accuracy : 0.6467          
                 95% CI : (0.6205, 0.6722)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 0.005317        
                                          
                  Kappa : 0.1256          
                                          
 Mcnemar's Test P-Value : < 2.2e-16       
                                          
            Sensitivity : 0.13958         
            Specificity : 0.96735         
         Pos Pred Value : 0.73000         
         Neg Pred Value : 0.64000         
             Prevalence : 0.38741         
         Detection Rate : 0.05407         
   Detection Prevalence : 0.07407         
      Balanced Accuracy : 0.55347         
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 230

compute_aupr(image_level$filter_230, image_level$label, 'PR: Segmentations: zero all pixels < 230')

compute_auroc(image_level$filter_230, image_level$label, 'ROC: Segmentations: zero all pixels < 230')

confusion_matrix(image_level, 'filter_230')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 464 160
         1 363 363
                                         
               Accuracy : 0.6126         
                 95% CI : (0.586, 0.6387)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : 0.512          
                                         
                  Kappa : 0.2381         
                                         
 Mcnemar's Test P-Value : <2e-16         
                                         
            Sensitivity : 0.6941         
            Specificity : 0.5611         
         Pos Pred Value : 0.5000         
         Neg Pred Value : 0.7436         
             Prevalence : 0.3874         
         Detection Rate : 0.2689         
   Detection Prevalence : 0.5378         
      Balanced Accuracy : 0.6276         
                                         
       'Positive' Class : 1              
                                         
compute_aupr(image_level$filter_230_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 230 and binarize')

compute_auroc(image_level$filter_230_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 230 and binarize')

confusion_matrix(image_level, 'filter_230_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 246  46
         1 581 477
                                          
               Accuracy : 0.5356          
                 95% CI : (0.5085, 0.5624)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 1               
                                          
                  Kappa : 0.1764          
                                          
 Mcnemar's Test P-Value : <2e-16          
                                          
            Sensitivity : 0.9120          
            Specificity : 0.2975          
         Pos Pred Value : 0.4509          
         Neg Pred Value : 0.8425          
             Prevalence : 0.3874          
         Detection Rate : 0.3533          
   Detection Prevalence : 0.7837          
      Balanced Accuracy : 0.6048          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_230_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 230 and skeletonize')

compute_auroc(image_level$filter_230_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 230 and skeletonize')

confusion_matrix(image_level, 'filter_230_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 365 110
         1 462 413
                                          
               Accuracy : 0.5763          
                 95% CI : (0.5494, 0.6028)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 0.997           
                                          
                  Kappa : 0.2056          
                                          
 Mcnemar's Test P-Value : <2e-16          
                                          
            Sensitivity : 0.7897          
            Specificity : 0.4414          
         Pos Pred Value : 0.4720          
         Neg Pred Value : 0.7684          
             Prevalence : 0.3874          
         Detection Rate : 0.3059          
   Detection Prevalence : 0.6481          
      Balanced Accuracy : 0.6155          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 240

compute_aupr(image_level$filter_240, image_level$label, 'PR: Segmentations: zero all pixels < 240')

compute_auroc(image_level$filter_240, image_level$label, 'ROC: Segmentations: zero all pixels < 240')

confusion_matrix(image_level, 'filter_240')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 414 183
         1 413 340
                                          
               Accuracy : 0.5585          
                 95% CI : (0.5316, 0.5852)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 1               
                                          
                  Kappa : 0.1394          
                                          
 Mcnemar's Test P-Value : <2e-16          
                                          
            Sensitivity : 0.6501          
            Specificity : 0.5006          
         Pos Pred Value : 0.4515          
         Neg Pred Value : 0.6935          
             Prevalence : 0.3874          
         Detection Rate : 0.2519          
   Detection Prevalence : 0.5578          
      Balanced Accuracy : 0.5754          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_240_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 240 and binarize')

compute_auroc(image_level$filter_240_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 240 and binarize')

confusion_matrix(image_level, 'filter_240_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 258  83
         1 569 440
                                       
               Accuracy : 0.517        
                 95% CI : (0.49, 0.544)
    No Information Rate : 0.6126       
    P-Value [Acc > NIR] : 1            
                                       
                  Kappa : 0.1309       
                                       
 Mcnemar's Test P-Value : <2e-16       
                                       
            Sensitivity : 0.8413       
            Specificity : 0.3120       
         Pos Pred Value : 0.4361       
         Neg Pred Value : 0.7566       
             Prevalence : 0.3874       
         Detection Rate : 0.3259       
   Detection Prevalence : 0.7474       
      Balanced Accuracy : 0.5766       
                                       
       'Positive' Class : 1            
                                       
compute_aupr(image_level$filter_240_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 240 and skeletonize')

compute_auroc(image_level$filter_240_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 240 and skeletonize')

confusion_matrix(image_level, 'filter_240_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 279 103
         1 548 420
                                          
               Accuracy : 0.5178          
                 95% CI : (0.4907, 0.5447)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 1               
                                          
                  Kappa : 0.1214          
                                          
 Mcnemar's Test P-Value : <2e-16          
                                          
            Sensitivity : 0.8031          
            Specificity : 0.3374          
         Pos Pred Value : 0.4339          
         Neg Pred Value : 0.7304          
             Prevalence : 0.3874          
         Detection Rate : 0.3111          
   Detection Prevalence : 0.7170          
      Balanced Accuracy : 0.5702          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 250

compute_aupr(image_level$filter_250, image_level$label, 'PR: Segmentations: zero all pixels < 250')

compute_auroc(image_level$filter_250, image_level$label, 'ROC: Segmentations: zero all pixels < 250')

confusion_matrix(image_level, 'filter_250')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0  51   9
         1 776 514
                                         
               Accuracy : 0.4185         
                 95% CI : (0.392, 0.4454)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : 1              
                                         
                  Kappa : 0.035          
                                         
 Mcnemar's Test P-Value : <2e-16         
                                         
            Sensitivity : 0.98279        
            Specificity : 0.06167        
         Pos Pred Value : 0.39845        
         Neg Pred Value : 0.85000        
             Prevalence : 0.38741        
         Detection Rate : 0.38074        
   Detection Prevalence : 0.95556        
      Balanced Accuracy : 0.52223        
                                         
       'Positive' Class : 1              
                                         
compute_aupr(image_level$filter_250_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 250 and binarize')

compute_auroc(image_level$filter_250_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 250 and binarize')

confusion_matrix(image_level, 'filter_250_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0  50   8
         1 777 515
                                         
               Accuracy : 0.4185         
                 95% CI : (0.392, 0.4454)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : 1              
                                         
                  Kappa : 0.0356         
                                         
 Mcnemar's Test P-Value : <2e-16         
                                         
            Sensitivity : 0.98470        
            Specificity : 0.06046        
         Pos Pred Value : 0.39861        
         Neg Pred Value : 0.86207        
             Prevalence : 0.38741        
         Detection Rate : 0.38148        
   Detection Prevalence : 0.95704        
      Balanced Accuracy : 0.52258        
                                         
       'Positive' Class : 1              
                                         
compute_aupr(image_level$filter_250_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 250 and skeletonize')

compute_auroc(image_level$filter_250_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 250 and skeletonize')

confusion_matrix(image_level, 'filter_250_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0  53   9
         1 774 514
                                          
               Accuracy : 0.42            
                 95% CI : (0.3935, 0.4468)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : 1               
                                          
                  Kappa : 0.0369          
                                          
 Mcnemar's Test P-Value : <2e-16          
                                          
            Sensitivity : 0.98279         
            Specificity : 0.06409         
         Pos Pred Value : 0.39907         
         Neg Pred Value : 0.85484         
             Prevalence : 0.38741         
         Detection Rate : 0.38074         
   Detection Prevalence : 0.95407         
      Balanced Accuracy : 0.52344         
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 257

compute_aupr(image_level$filter_257, image_level$label, 'PR: Segmentations: zero all pixels < 257')

compute_auroc(image_level$filter_257, image_level$label, 'ROC: Segmentations: zero all pixels < 257')

confusion_matrix(image_level, 'filter_257')
Warning in confusionMatrix.default(preds, df$label, positive = "1") :
  Levels are not in the same order for reference and data. Refactoring data to match.
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 827 523
         1   0   0
                                         
               Accuracy : 0.6126         
                 95% CI : (0.586, 0.6387)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : 0.512          
                                         
                  Kappa : 0              
                                         
 Mcnemar's Test P-Value : <2e-16         
                                         
            Sensitivity : 0.0000         
            Specificity : 1.0000         
         Pos Pred Value :    NaN         
         Neg Pred Value : 0.6126         
             Prevalence : 0.3874         
         Detection Rate : 0.0000         
   Detection Prevalence : 0.0000         
      Balanced Accuracy : 0.5000         
                                         
       'Positive' Class : 1              
                                         
compute_aupr(image_level$filter_257_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 257 and binarize')

compute_auroc(image_level$filter_257_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 257 and binarize')

confusion_matrix(image_level, 'filter_257_binarized')
Warning in confusionMatrix.default(preds, df$label, positive = "1") :
  Levels are not in the same order for reference and data. Refactoring data to match.
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 827 523
         1   0   0
                                         
               Accuracy : 0.6126         
                 95% CI : (0.586, 0.6387)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : 0.512          
                                         
                  Kappa : 0              
                                         
 Mcnemar's Test P-Value : <2e-16         
                                         
            Sensitivity : 0.0000         
            Specificity : 1.0000         
         Pos Pred Value :    NaN         
         Neg Pred Value : 0.6126         
             Prevalence : 0.3874         
         Detection Rate : 0.0000         
   Detection Prevalence : 0.0000         
      Balanced Accuracy : 0.5000         
                                         
       'Positive' Class : 1              
                                         
compute_aupr(image_level$filter_257_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 257 and skeletonize')

compute_auroc(image_level$filter_257_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 257 and skeletonize')

confusion_matrix(image_level, 'filter_257_skeletonized')
Warning in confusionMatrix.default(preds, df$label, positive = "1") :
  Levels are not in the same order for reference and data. Refactoring data to match.
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 827 523
         1   0   0
                                         
               Accuracy : 0.6126         
                 95% CI : (0.586, 0.6387)
    No Information Rate : 0.6126         
    P-Value [Acc > NIR] : 0.512          
                                         
                  Kappa : 0              
                                         
 Mcnemar's Test P-Value : <2e-16         
                                         
            Sensitivity : 0.0000         
            Specificity : 1.0000         
         Pos Pred Value :    NaN         
         Neg Pred Value : 0.6126         
             Prevalence : 0.3874         
         Detection Rate : 0.0000         
   Detection Prevalence : 0.0000         
      Balanced Accuracy : 0.5000         
                                         
       'Positive' Class : 1              
                                         

Segmentations: zero all pixels > 10

compute_aupr(image_level$filter_10, image_level$label, 'PR: Segmentations: zero all pixels < 10')

compute_auroc(image_level$filter_10, image_level$label, 'ROC: Segmentations: zero all pixels < 10')

confusion_matrix(image_level, 'filter_10')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 763 196
         1  64 327
                                          
               Accuracy : 0.8074          
                 95% CI : (0.7853, 0.8281)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2e-16         
                                          
                  Kappa : 0.5745          
                                          
 Mcnemar's Test P-Value : 4.5e-16         
                                          
            Sensitivity : 0.6252          
            Specificity : 0.9226          
         Pos Pred Value : 0.8363          
         Neg Pred Value : 0.7956          
             Prevalence : 0.3874          
         Detection Rate : 0.2422          
   Detection Prevalence : 0.2896          
      Balanced Accuracy : 0.7739          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_10_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 10 and binarize')

compute_auroc(image_level$filter_10_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 10 and binarize')

confusion_matrix(image_level, 'filter_10_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 621   7
         1 206 516
                                          
               Accuracy : 0.8422          
                 95% CI : (0.8217, 0.8613)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.6893          
                                          
 Mcnemar's Test P-Value : < 2.2e-16       
                                          
            Sensitivity : 0.9866          
            Specificity : 0.7509          
         Pos Pred Value : 0.7147          
         Neg Pred Value : 0.9889          
             Prevalence : 0.3874          
         Detection Rate : 0.3822          
   Detection Prevalence : 0.5348          
      Balanced Accuracy : 0.8688          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_10_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 10 and skeletonize')

compute_auroc(image_level$filter_10_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 10 and skeletonize')

confusion_matrix(image_level, 'filter_10_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 696  43
         1 131 480
                                          
               Accuracy : 0.8711          
                 95% CI : (0.8521, 0.8885)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.7366          
                                          
 Mcnemar's Test P-Value : 4.24e-11        
                                          
            Sensitivity : 0.9178          
            Specificity : 0.8416          
         Pos Pred Value : 0.7856          
         Neg Pred Value : 0.9418          
             Prevalence : 0.3874          
         Detection Rate : 0.3556          
   Detection Prevalence : 0.4526          
      Balanced Accuracy : 0.8797          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 75 and > 150

compute_aupr(image_level$filter_75, image_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150')

compute_auroc(image_level$filter_75, image_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150')

confusion_matrix(image_level, 'filter_75')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 597  77
         1 230 446
                                          
               Accuracy : 0.7726          
                 95% CI : (0.7493, 0.7947)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.5453          
                                          
 Mcnemar's Test P-Value : < 2.2e-16       
                                          
            Sensitivity : 0.8528          
            Specificity : 0.7219          
         Pos Pred Value : 0.6598          
         Neg Pred Value : 0.8858          
             Prevalence : 0.3874          
         Detection Rate : 0.3304          
   Detection Prevalence : 0.5007          
      Balanced Accuracy : 0.7873          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_75_binarized, image_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150 and binarize')

compute_auroc(image_level$filter_75_binarized, image_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150 and binarize')

confusion_matrix(image_level, 'filter_75_binarized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 545  69
         1 282 454
                                          
               Accuracy : 0.74            
                 95% CI : (0.7157, 0.7632)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.4904          
                                          
 Mcnemar's Test P-Value : < 2.2e-16       
                                          
            Sensitivity : 0.8681          
            Specificity : 0.6590          
         Pos Pred Value : 0.6168          
         Neg Pred Value : 0.8876          
             Prevalence : 0.3874          
         Detection Rate : 0.3363          
   Detection Prevalence : 0.5452          
      Balanced Accuracy : 0.7635          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(image_level$filter_75_skeletonized, image_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150 and skeletonize')

compute_auroc(image_level$filter_75_skeletonized, image_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150 and skeletonize')

confusion_matrix(image_level, 'filter_75_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 726 224
         1 101 299
                                          
               Accuracy : 0.7593          
                 95% CI : (0.7355, 0.7819)
    No Information Rate : 0.6126          
    P-Value [Acc > NIR] : < 2.2e-16       
                                          
                  Kappa : 0.4699          
                                          
 Mcnemar's Test P-Value : 1.312e-11       
                                          
            Sensitivity : 0.5717          
            Specificity : 0.8779          
         Pos Pred Value : 0.7475          
         Neg Pred Value : 0.7642          
             Prevalence : 0.3874          
         Detection Rate : 0.2215          
   Detection Prevalence : 0.2963          
      Balanced Accuracy : 0.7248          
                                          
       'Positive' Class : 1               
                                          

Subject-level Analysis

RetCam Images

compute_aupr(subject_level$retcam, subject_level$label, 'PR: Raw RetCam Images')

compute_auroc(subject_level$retcam, subject_level$label, 'ROC: Raw RetCam Images')

confusion_matrix(subject_level, 'retcam')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43  0
         1  0 27
                                     
               Accuracy : 1          
                 95% CI : (0.9487, 1)
    No Information Rate : 0.6143     
    P-Value [Acc > NIR] : 1.534e-15  
                                     
                  Kappa : 1          
                                     
 Mcnemar's Test P-Value : NA         
                                     
            Sensitivity : 1.0000     
            Specificity : 1.0000     
         Pos Pred Value : 1.0000     
         Neg Pred Value : 1.0000     
             Prevalence : 0.3857     
         Detection Rate : 0.3857     
   Detection Prevalence : 0.3857     
      Balanced Accuracy : 1.0000     
                                     
       'Positive' Class : 1          
                                     
compute_aupr(subject_level$retcam_random, subject_level$label, 'PR: Raw RetCam Images - Shuffled Labels')

compute_auroc(subject_level$retcam_random, subject_level$label, 'ROC: Raw RetCam Images - Shuffled Labels')

confusion_matrix(subject_level, 'retcam_random')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 24  2
         1 19 25
                                          
               Accuracy : 0.7             
                 95% CI : (0.5787, 0.8038)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.0869051       
                                          
                  Kappa : 0.4333          
                                          
 Mcnemar's Test P-Value : 0.0004803       
                                          
            Sensitivity : 0.9259          
            Specificity : 0.5581          
         Pos Pred Value : 0.5682          
         Neg Pred Value : 0.9231          
             Prevalence : 0.3857          
         Detection Rate : 0.3571          
   Detection Prevalence : 0.6286          
      Balanced Accuracy : 0.7420          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 0

compute_aupr(subject_level$filter_0, subject_level$label, 'PR: Segmentations: zero all pixels < 0')

compute_auroc(subject_level$filter_0, subject_level$label, 'ROC: Segmentations: zero all pixels < 0')

confusion_matrix(subject_level, 'filter_0')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 42  2
         1  1 25
                                          
               Accuracy : 0.9571          
                 95% CI : (0.8798, 0.9911)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 2.232e-11       
                                          
                  Kappa : 0.9089          
                                          
 Mcnemar's Test P-Value : 1               
                                          
            Sensitivity : 0.9259          
            Specificity : 0.9767          
         Pos Pred Value : 0.9615          
         Neg Pred Value : 0.9545          
             Prevalence : 0.3857          
         Detection Rate : 0.3571          
   Detection Prevalence : 0.3714          
      Balanced Accuracy : 0.9513          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_0_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 0 and binarize')

compute_auroc(subject_level$filter_0_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 0 and binarize')

confusion_matrix(subject_level, 'filter_0_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 41  0
         1  2 27
                                          
               Accuracy : 0.9714          
                 95% CI : (0.9006, 0.9965)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 1.53e-12        
                                          
                  Kappa : 0.9405          
                                          
 Mcnemar's Test P-Value : 0.4795          
                                          
            Sensitivity : 1.0000          
            Specificity : 0.9535          
         Pos Pred Value : 0.9310          
         Neg Pred Value : 1.0000          
             Prevalence : 0.3857          
         Detection Rate : 0.3857          
   Detection Prevalence : 0.4143          
      Balanced Accuracy : 0.9767          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_0_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 0 and skeletonize')

compute_auroc(subject_level$filter_0_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 0 and skeletonize')

confusion_matrix(subject_level, 'filter_0_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 40  0
         1  3 27
                                          
               Accuracy : 0.9571          
                 95% CI : (0.8798, 0.9911)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 2.232e-11       
                                          
                  Kappa : 0.9114          
                                          
 Mcnemar's Test P-Value : 0.2482          
                                          
            Sensitivity : 1.0000          
            Specificity : 0.9302          
         Pos Pred Value : 0.9000          
         Neg Pred Value : 1.0000          
             Prevalence : 0.3857          
         Detection Rate : 0.3857          
   Detection Prevalence : 0.4286          
      Balanced Accuracy : 0.9651          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_0_random, subject_level$label, 'PR: Segmentations: zero all pixels < 0 - Shuffled Labels')

compute_auroc(subject_level$filter_0_random, subject_level$label, 'ROC: Segmentations: zero all pixels < 0 - Shuffled Labels')

confusion_matrix(subject_level, 'filter_0_random')
Warning in confusionMatrix.default(preds, df$label, positive = "1") :
  Levels are not in the same order for reference and data. Refactoring data to match.
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43 27
         1  0  0
                                          
               Accuracy : 0.6143          
                 95% CI : (0.4903, 0.7283)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.5525          
                                          
                  Kappa : 0               
                                          
 Mcnemar's Test P-Value : 5.624e-07       
                                          
            Sensitivity : 0.0000          
            Specificity : 1.0000          
         Pos Pred Value :    NaN          
         Neg Pred Value : 0.6143          
             Prevalence : 0.3857          
         Detection Rate : 0.0000          
   Detection Prevalence : 0.0000          
      Balanced Accuracy : 0.5000          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 50

compute_aupr(subject_level$filter_50, subject_level$label, 'PR: Segmentations: zero all pixels < 50')

compute_auroc(subject_level$filter_50, subject_level$label, 'ROC: Segmentations: zero all pixels < 50')

confusion_matrix(subject_level, 'filter_50')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 36  0
         1  7 27
                                          
               Accuracy : 0.9             
                 95% CI : (0.8048, 0.9588)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 8.517e-08       
                                          
                  Kappa : 0.7987          
                                          
 Mcnemar's Test P-Value : 0.02334         
                                          
            Sensitivity : 1.0000          
            Specificity : 0.8372          
         Pos Pred Value : 0.7941          
         Neg Pred Value : 1.0000          
             Prevalence : 0.3857          
         Detection Rate : 0.3857          
   Detection Prevalence : 0.4857          
      Balanced Accuracy : 0.9186          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_50_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 50 and binarize')

compute_auroc(subject_level$filter_50_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 50 and binarize')

confusion_matrix(subject_level, 'filter_0_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 41  0
         1  2 27
                                          
               Accuracy : 0.9714          
                 95% CI : (0.9006, 0.9965)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 1.53e-12        
                                          
                  Kappa : 0.9405          
                                          
 Mcnemar's Test P-Value : 0.4795          
                                          
            Sensitivity : 1.0000          
            Specificity : 0.9535          
         Pos Pred Value : 0.9310          
         Neg Pred Value : 1.0000          
             Prevalence : 0.3857          
         Detection Rate : 0.3857          
   Detection Prevalence : 0.4143          
      Balanced Accuracy : 0.9767          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_50_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 50 and skeletonize')

compute_auroc(subject_level$filter_50_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 50 and skeletonize')

confusion_matrix(subject_level, 'filter_50_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 19  0
         1 24 27
                                         
               Accuracy : 0.6571         
                 95% CI : (0.534, 0.7665)
    No Information Rate : 0.6143         
    P-Value [Acc > NIR] : 0.2718         
                                         
                  Kappa : 0.3792         
                                         
 Mcnemar's Test P-Value : 2.668e-06      
                                         
            Sensitivity : 1.0000         
            Specificity : 0.4419         
         Pos Pred Value : 0.5294         
         Neg Pred Value : 1.0000         
             Prevalence : 0.3857         
         Detection Rate : 0.3857         
   Detection Prevalence : 0.7286         
      Balanced Accuracy : 0.7209         
                                         
       'Positive' Class : 1              
                                         

Segmentations: zero all pixels < 100

compute_aupr(subject_level$filter_100, subject_level$label, 'PR: Segmentations: zero all pixels < 100')

compute_auroc(subject_level$filter_100, subject_level$label, 'ROC: Segmentations: zero all pixels < 100')

confusion_matrix(subject_level, 'filter_100')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43 10
         1  0 17
                                          
               Accuracy : 0.8571          
                 95% CI : (0.7529, 0.9293)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 7.748e-06       
                                          
                  Kappa : 0.6762          
                                          
 Mcnemar's Test P-Value : 0.004427        
                                          
            Sensitivity : 0.6296          
            Specificity : 1.0000          
         Pos Pred Value : 1.0000          
         Neg Pred Value : 0.8113          
             Prevalence : 0.3857          
         Detection Rate : 0.2429          
   Detection Prevalence : 0.2429          
      Balanced Accuracy : 0.8148          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_100_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 100 and binarize')

compute_auroc(subject_level$filter_100_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 100 and binarize')

confusion_matrix(subject_level, 'filter_100_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 39  4
         1  4 23
                                          
               Accuracy : 0.8857          
                 95% CI : (0.7872, 0.9493)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 4.352e-07       
                                          
                  Kappa : 0.7588          
                                          
 Mcnemar's Test P-Value : 1               
                                          
            Sensitivity : 0.8519          
            Specificity : 0.9070          
         Pos Pred Value : 0.8519          
         Neg Pred Value : 0.9070          
             Prevalence : 0.3857          
         Detection Rate : 0.3286          
   Detection Prevalence : 0.3857          
      Balanced Accuracy : 0.8794          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_100_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 100 and skeletonize')

compute_auroc(subject_level$filter_100_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 100 and skeletonize')

confusion_matrix(subject_level, 'filter_100_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 38  3
         1  5 24
                                          
               Accuracy : 0.8857          
                 95% CI : (0.7872, 0.9493)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 4.352e-07       
                                          
                  Kappa : 0.7621          
                                          
 Mcnemar's Test P-Value : 0.7237          
                                          
            Sensitivity : 0.8889          
            Specificity : 0.8837          
         Pos Pred Value : 0.8276          
         Neg Pred Value : 0.9268          
             Prevalence : 0.3857          
         Detection Rate : 0.3429          
   Detection Prevalence : 0.4143          
      Balanced Accuracy : 0.8863          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 150

compute_aupr(subject_level$filter_150, subject_level$label, 'PR: Segmentations: zero all pixels < 150')

compute_auroc(subject_level$filter_150, subject_level$label, 'ROC: Segmentations: zero all pixels < 150')

confusion_matrix(subject_level, 'filter_150')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 37  0
         1  6 27
                                          
               Accuracy : 0.9143          
                 95% CI : (0.8227, 0.9679)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 1.438e-08       
                                          
                  Kappa : 0.8263          
                                          
 Mcnemar's Test P-Value : 0.04123         
                                          
            Sensitivity : 1.0000          
            Specificity : 0.8605          
         Pos Pred Value : 0.8182          
         Neg Pred Value : 1.0000          
             Prevalence : 0.3857          
         Detection Rate : 0.3857          
   Detection Prevalence : 0.4714          
      Balanced Accuracy : 0.9302          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_150_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 150 and binarize')

compute_auroc(subject_level$filter_150_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 150 and binarize')

confusion_matrix(subject_level, 'filter_150_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43  5
         1  0 22
                                          
               Accuracy : 0.9286          
                 95% CI : (0.8411, 0.9764)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 2.054e-09       
                                          
                  Kappa : 0.8439          
                                          
 Mcnemar's Test P-Value : 0.07364         
                                          
            Sensitivity : 0.8148          
            Specificity : 1.0000          
         Pos Pred Value : 1.0000          
         Neg Pred Value : 0.8958          
             Prevalence : 0.3857          
         Detection Rate : 0.3143          
   Detection Prevalence : 0.3143          
      Balanced Accuracy : 0.9074          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_150_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 150 and skeletonize')

compute_auroc(subject_level$filter_150_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 150 and skeletonize')

confusion_matrix(subject_level, 'filter_150_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 38  3
         1  5 24
                                          
               Accuracy : 0.8857          
                 95% CI : (0.7872, 0.9493)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 4.352e-07       
                                          
                  Kappa : 0.7621          
                                          
 Mcnemar's Test P-Value : 0.7237          
                                          
            Sensitivity : 0.8889          
            Specificity : 0.8837          
         Pos Pred Value : 0.8276          
         Neg Pred Value : 0.9268          
             Prevalence : 0.3857          
         Detection Rate : 0.3429          
   Detection Prevalence : 0.4143          
      Balanced Accuracy : 0.8863          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 200

compute_aupr(subject_level$filter_200, subject_level$label, 'PR: Segmentations: zero all pixels < 200')

compute_auroc(subject_level$filter_200, subject_level$label, 'ROC: Segmentations: zero all pixels < 200')

confusion_matrix(subject_level, 'filter_200')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 34  4
         1  9 23
                                          
               Accuracy : 0.8143          
                 95% CI : (0.7034, 0.8972)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.0002607       
                                          
                  Kappa : 0.6211          
                                          
 Mcnemar's Test P-Value : 0.2672575       
                                          
            Sensitivity : 0.8519          
            Specificity : 0.7907          
         Pos Pred Value : 0.7187          
         Neg Pred Value : 0.8947          
             Prevalence : 0.3857          
         Detection Rate : 0.3286          
   Detection Prevalence : 0.4571          
      Balanced Accuracy : 0.8213          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_200_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 200 and binarize')

compute_auroc(subject_level$filter_200_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 200 and binarize')

confusion_matrix(subject_level, 'filter_200_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 40 12
         1  3 15
                                          
               Accuracy : 0.7857          
                 95% CI : (0.6713, 0.8748)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.001729        
                                          
                  Kappa : 0.5179          
                                          
 Mcnemar's Test P-Value : 0.038867        
                                          
            Sensitivity : 0.5556          
            Specificity : 0.9302          
         Pos Pred Value : 0.8333          
         Neg Pred Value : 0.7692          
             Prevalence : 0.3857          
         Detection Rate : 0.2143          
   Detection Prevalence : 0.2571          
      Balanced Accuracy : 0.7429          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_200_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 200 and skeletonize')

compute_auroc(subject_level$filter_200_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 200 and skeletonize')

confusion_matrix(subject_level, 'filter_200_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43 25
         1  0  2
                                          
               Accuracy : 0.6429          
                 95% CI : (0.5193, 0.7539)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.3595          
                                          
                  Kappa : 0.0895          
                                          
 Mcnemar's Test P-Value : 1.587e-06       
                                          
            Sensitivity : 0.07407         
            Specificity : 1.00000         
         Pos Pred Value : 1.00000         
         Neg Pred Value : 0.63235         
             Prevalence : 0.38571         
         Detection Rate : 0.02857         
   Detection Prevalence : 0.02857         
      Balanced Accuracy : 0.53704         
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 210

compute_aupr(subject_level$filter_210, subject_level$label, 'PR: Segmentations: zero all pixels < 210')

compute_auroc(subject_level$filter_210, subject_level$label, 'ROC: Segmentations: zero all pixels < 210')

confusion_matrix(subject_level, 'filter_210')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43 20
         1  0  7
                                         
               Accuracy : 0.7143         
                 95% CI : (0.5938, 0.816)
    No Information Rate : 0.6143         
    P-Value [Acc > NIR] : 0.0532         
                                         
                  Kappa : 0.3007         
                                         
 Mcnemar's Test P-Value : 2.152e-05      
                                         
            Sensitivity : 0.2593         
            Specificity : 1.0000         
         Pos Pred Value : 1.0000         
         Neg Pred Value : 0.6825         
             Prevalence : 0.3857         
         Detection Rate : 0.1000         
   Detection Prevalence : 0.1000         
      Balanced Accuracy : 0.6296         
                                         
       'Positive' Class : 1              
                                         
compute_aupr(subject_level$filter_210_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 210 and binarize')

compute_auroc(subject_level$filter_210_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 210 and binarize')

confusion_matrix(subject_level, 'filter_210_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 39  9
         1  4 18
                                          
               Accuracy : 0.8143          
                 95% CI : (0.7034, 0.8972)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.0002607       
                                          
                  Kappa : 0.5941          
                                          
 Mcnemar's Test P-Value : 0.2672575       
                                          
            Sensitivity : 0.6667          
            Specificity : 0.9070          
         Pos Pred Value : 0.8182          
         Neg Pred Value : 0.8125          
             Prevalence : 0.3857          
         Detection Rate : 0.2571          
   Detection Prevalence : 0.3143          
      Balanced Accuracy : 0.7868          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_210_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 210 and skeletonize')

compute_auroc(subject_level$filter_210_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 210 and skeletonize')

confusion_matrix(subject_level, 'filter_210_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 35  7
         1  8 20
                                          
               Accuracy : 0.7857          
                 95% CI : (0.6713, 0.8748)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.001729        
                                          
                  Kappa : 0.5509          
                                          
 Mcnemar's Test P-Value : 1.000000        
                                          
            Sensitivity : 0.7407          
            Specificity : 0.8140          
         Pos Pred Value : 0.7143          
         Neg Pred Value : 0.8333          
             Prevalence : 0.3857          
         Detection Rate : 0.2857          
   Detection Prevalence : 0.4000          
      Balanced Accuracy : 0.7773          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 220

compute_aupr(subject_level$filter_220, subject_level$label, 'PR: Segmentations: zero all pixels < 220')

compute_auroc(subject_level$filter_220, subject_level$label, 'ROC: Segmentations: zero all pixels < 220')

confusion_matrix(subject_level, 'filter_220')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 34 12
         1  9 15
                                          
               Accuracy : 0.7             
                 95% CI : (0.5787, 0.8038)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.08691         
                                          
                  Kappa : 0.3536          
                                          
 Mcnemar's Test P-Value : 0.66252         
                                          
            Sensitivity : 0.5556          
            Specificity : 0.7907          
         Pos Pred Value : 0.6250          
         Neg Pred Value : 0.7391          
             Prevalence : 0.3857          
         Detection Rate : 0.2143          
   Detection Prevalence : 0.3429          
      Balanced Accuracy : 0.6731          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_220_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 220 and binarize')

compute_auroc(subject_level$filter_220_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 220 and binarize')

confusion_matrix(subject_level, 'filter_220_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 31  7
         1 12 20
                                        
               Accuracy : 0.7286        
                 95% CI : (0.609, 0.828)
    No Information Rate : 0.6143        
    P-Value [Acc > NIR] : 0.03066       
                                        
                  Kappa : 0.4463        
                                        
 Mcnemar's Test P-Value : 0.35880       
                                        
            Sensitivity : 0.7407        
            Specificity : 0.7209        
         Pos Pred Value : 0.6250        
         Neg Pred Value : 0.8158        
             Prevalence : 0.3857        
         Detection Rate : 0.2857        
   Detection Prevalence : 0.4571        
      Balanced Accuracy : 0.7308        
                                        
       'Positive' Class : 1             
                                        
compute_aupr(subject_level$filter_220_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 220 and skeletonize')

compute_auroc(subject_level$filter_220_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 220 and skeletonize')

confusion_matrix(subject_level, 'filter_220_skeletonized')
Warning in confusionMatrix.default(preds, df$label, positive = "1") :
  Levels are not in the same order for reference and data. Refactoring data to match.
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43 27
         1  0  0
                                          
               Accuracy : 0.6143          
                 95% CI : (0.4903, 0.7283)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.5525          
                                          
                  Kappa : 0               
                                          
 Mcnemar's Test P-Value : 5.624e-07       
                                          
            Sensitivity : 0.0000          
            Specificity : 1.0000          
         Pos Pred Value :    NaN          
         Neg Pred Value : 0.6143          
             Prevalence : 0.3857          
         Detection Rate : 0.0000          
   Detection Prevalence : 0.0000          
      Balanced Accuracy : 0.5000          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 230

compute_aupr(subject_level$filter_230, subject_level$label, 'PR: Segmentations: zero all pixels < 230')

compute_auroc(subject_level$filter_230, subject_level$label, 'ROC: Segmentations: zero all pixels < 230')

confusion_matrix(subject_level, 'filter_230')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 26  6
         1 17 21
                                          
               Accuracy : 0.6714          
                 95% CI : (0.5488, 0.7791)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.19581         
                                          
                  Kappa : 0.3555          
                                          
 Mcnemar's Test P-Value : 0.03706         
                                          
            Sensitivity : 0.7778          
            Specificity : 0.6047          
         Pos Pred Value : 0.5526          
         Neg Pred Value : 0.8125          
             Prevalence : 0.3857          
         Detection Rate : 0.3000          
   Detection Prevalence : 0.5429          
      Balanced Accuracy : 0.6912          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_230_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 230 and binarize')

compute_auroc(subject_level$filter_230_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 230 and binarize')

confusion_matrix(subject_level, 'filter_230_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 10  1
         1 33 26
                                          
               Accuracy : 0.5143          
                 95% CI : (0.3917, 0.6356)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.966           
                                          
                  Kappa : 0.1602          
                                          
 Mcnemar's Test P-Value : 1.058e-07       
                                          
            Sensitivity : 0.9630          
            Specificity : 0.2326          
         Pos Pred Value : 0.4407          
         Neg Pred Value : 0.9091          
             Prevalence : 0.3857          
         Detection Rate : 0.3714          
   Detection Prevalence : 0.8429          
      Balanced Accuracy : 0.5978          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_230_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 230 and skeletonize')

compute_auroc(subject_level$filter_230_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 230 and skeletonize')

confusion_matrix(subject_level, 'filter_230_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 17  2
         1 26 25
                                          
               Accuracy : 0.6             
                 95% CI : (0.4759, 0.7153)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.6466          
                                          
                  Kappa : 0.2757          
                                          
 Mcnemar's Test P-Value : 1.383e-05       
                                          
            Sensitivity : 0.9259          
            Specificity : 0.3953          
         Pos Pred Value : 0.4902          
         Neg Pred Value : 0.8947          
             Prevalence : 0.3857          
         Detection Rate : 0.3571          
   Detection Prevalence : 0.7286          
      Balanced Accuracy : 0.6606          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 240

compute_aupr(subject_level$filter_240, subject_level$label, 'PR: Segmentations: zero all pixels < 240')

compute_auroc(subject_level$filter_240, subject_level$label, 'ROC: Segmentations: zero all pixels < 240')

confusion_matrix(subject_level, 'filter_240')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 23  9
         1 20 18
                                          
               Accuracy : 0.5857          
                 95% CI : (0.4617, 0.7023)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.73214         
                                          
                  Kappa : 0.1873          
                                          
 Mcnemar's Test P-Value : 0.06332         
                                          
            Sensitivity : 0.6667          
            Specificity : 0.5349          
         Pos Pred Value : 0.4737          
         Neg Pred Value : 0.7187          
             Prevalence : 0.3857          
         Detection Rate : 0.2571          
   Detection Prevalence : 0.5429          
      Balanced Accuracy : 0.6008          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_240_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 240 and binarize')

compute_auroc(subject_level$filter_240_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 240 and binarize')

confusion_matrix(subject_level, 'filter_240_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 12  1
         1 31 26
                                          
               Accuracy : 0.5429          
                 95% CI : (0.4194, 0.6626)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.9106          
                                          
                  Kappa : 0.2006          
                                          
 Mcnemar's Test P-Value : 2.951e-07       
                                          
            Sensitivity : 0.9630          
            Specificity : 0.2791          
         Pos Pred Value : 0.4561          
         Neg Pred Value : 0.9231          
             Prevalence : 0.3857          
         Detection Rate : 0.3714          
   Detection Prevalence : 0.8143          
      Balanced Accuracy : 0.6210          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_240_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 240 and skeletonize')

compute_auroc(subject_level$filter_240_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 240 and skeletonize')

confusion_matrix(subject_level, 'filter_240_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 14  2
         1 29 25
                                          
               Accuracy : 0.5571          
                 95% CI : (0.4334, 0.6759)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.8651          
                                          
                  Kappa : 0.2121          
                                          
 Mcnemar's Test P-Value : 3.016e-06       
                                          
            Sensitivity : 0.9259          
            Specificity : 0.3256          
         Pos Pred Value : 0.4630          
         Neg Pred Value : 0.8750          
             Prevalence : 0.3857          
         Detection Rate : 0.3571          
   Detection Prevalence : 0.7714          
      Balanced Accuracy : 0.6258          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 250

compute_aupr(subject_level$filter_250, subject_level$label, 'PR: Segmentations: zero all pixels < 250')

compute_auroc(subject_level$filter_250, subject_level$label, 'ROC: Segmentations: zero all pixels < 250')

confusion_matrix(subject_level, 'filter_250')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0  1  0
         1 42 27
                                          
               Accuracy : 0.4             
                 95% CI : (0.2847, 0.5241)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.9999          
                                          
                  Kappa : 0.018           
                                          
 Mcnemar's Test P-Value : 2.509e-10       
                                          
            Sensitivity : 1.00000         
            Specificity : 0.02326         
         Pos Pred Value : 0.39130         
         Neg Pred Value : 1.00000         
             Prevalence : 0.38571         
         Detection Rate : 0.38571         
   Detection Prevalence : 0.98571         
      Balanced Accuracy : 0.51163         
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_250_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 250 and binarize')

compute_auroc(subject_level$filter_250_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 250 and binarize')

confusion_matrix(subject_level, 'filter_250_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0  1  0
         1 42 27
                                          
               Accuracy : 0.4             
                 95% CI : (0.2847, 0.5241)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.9999          
                                          
                  Kappa : 0.018           
                                          
 Mcnemar's Test P-Value : 2.509e-10       
                                          
            Sensitivity : 1.00000         
            Specificity : 0.02326         
         Pos Pred Value : 0.39130         
         Neg Pred Value : 1.00000         
             Prevalence : 0.38571         
         Detection Rate : 0.38571         
   Detection Prevalence : 0.98571         
      Balanced Accuracy : 0.51163         
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_250_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 250 and skeletonize')

compute_auroc(subject_level$filter_250_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 250 and skeletonize')

confusion_matrix(subject_level, 'filter_250_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0  1  0
         1 42 27
                                          
               Accuracy : 0.4             
                 95% CI : (0.2847, 0.5241)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.9999          
                                          
                  Kappa : 0.018           
                                          
 Mcnemar's Test P-Value : 2.509e-10       
                                          
            Sensitivity : 1.00000         
            Specificity : 0.02326         
         Pos Pred Value : 0.39130         
         Neg Pred Value : 1.00000         
             Prevalence : 0.38571         
         Detection Rate : 0.38571         
   Detection Prevalence : 0.98571         
      Balanced Accuracy : 0.51163         
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 257

compute_aupr(subject_level$filter_257, subject_level$label, 'PR: Segmentations: zero all pixels < 257')

compute_auroc(subject_level$filter_257, subject_level$label, 'ROC: Segmentations: zero all pixels < 257')

confusion_matrix(subject_level, 'filter_257')
Warning in confusionMatrix.default(preds, df$label, positive = "1") :
  Levels are not in the same order for reference and data. Refactoring data to match.
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43 27
         1  0  0
                                          
               Accuracy : 0.6143          
                 95% CI : (0.4903, 0.7283)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.5525          
                                          
                  Kappa : 0               
                                          
 Mcnemar's Test P-Value : 5.624e-07       
                                          
            Sensitivity : 0.0000          
            Specificity : 1.0000          
         Pos Pred Value :    NaN          
         Neg Pred Value : 0.6143          
             Prevalence : 0.3857          
         Detection Rate : 0.0000          
   Detection Prevalence : 0.0000          
      Balanced Accuracy : 0.5000          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_257_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 257 and binarize')

compute_auroc(subject_level$filter_257_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 257 and binarize')

confusion_matrix(subject_level, 'filter_257_binarized')
Warning in confusionMatrix.default(preds, df$label, positive = "1") :
  Levels are not in the same order for reference and data. Refactoring data to match.
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43 27
         1  0  0
                                          
               Accuracy : 0.6143          
                 95% CI : (0.4903, 0.7283)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.5525          
                                          
                  Kappa : 0               
                                          
 Mcnemar's Test P-Value : 5.624e-07       
                                          
            Sensitivity : 0.0000          
            Specificity : 1.0000          
         Pos Pred Value :    NaN          
         Neg Pred Value : 0.6143          
             Prevalence : 0.3857          
         Detection Rate : 0.0000          
   Detection Prevalence : 0.0000          
      Balanced Accuracy : 0.5000          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_257_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 257 and skeletonize')

compute_auroc(subject_level$filter_257_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 257 and skeletonize')

confusion_matrix(subject_level, 'filter_257_skeletonized')
Warning in confusionMatrix.default(preds, df$label, positive = "1") :
  Levels are not in the same order for reference and data. Refactoring data to match.
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43 27
         1  0  0
                                          
               Accuracy : 0.6143          
                 95% CI : (0.4903, 0.7283)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.5525          
                                          
                  Kappa : 0               
                                          
 Mcnemar's Test P-Value : 5.624e-07       
                                          
            Sensitivity : 0.0000          
            Specificity : 1.0000          
         Pos Pred Value :    NaN          
         Neg Pred Value : 0.6143          
             Prevalence : 0.3857          
         Detection Rate : 0.0000          
   Detection Prevalence : 0.0000          
      Balanced Accuracy : 0.5000          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels > 10

compute_aupr(subject_level$filter_10, subject_level$label, 'PR: Segmentations: zero all pixels < 10')

compute_auroc(subject_level$filter_10, subject_level$label, 'ROC: Segmentations: zero all pixels < 10')

confusion_matrix(subject_level, 'filter_10')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 41  8
         1  2 19
                                          
               Accuracy : 0.8571          
                 95% CI : (0.7529, 0.9293)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 7.748e-06       
                                          
                  Kappa : 0.6855          
                                          
 Mcnemar's Test P-Value : 0.1138          
                                          
            Sensitivity : 0.7037          
            Specificity : 0.9535          
         Pos Pred Value : 0.9048          
         Neg Pred Value : 0.8367          
             Prevalence : 0.3857          
         Detection Rate : 0.2714          
   Detection Prevalence : 0.3000          
      Balanced Accuracy : 0.8286          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_10_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 10 and binarize')

compute_auroc(subject_level$filter_10_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 10 and binarize')

confusion_matrix(subject_level, 'filter_10_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 34  0
         1  9 27
                                          
               Accuracy : 0.8714          
                 95% CI : (0.7699, 0.9395)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 1.949e-06       
                                          
                  Kappa : 0.7445          
                                          
 Mcnemar's Test P-Value : 0.007661        
                                          
            Sensitivity : 1.0000          
            Specificity : 0.7907          
         Pos Pred Value : 0.7500          
         Neg Pred Value : 1.0000          
             Prevalence : 0.3857          
         Detection Rate : 0.3857          
   Detection Prevalence : 0.5143          
      Balanced Accuracy : 0.8953          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_10_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 10 and skeletonize')

compute_auroc(subject_level$filter_10_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 10 and skeletonize')

confusion_matrix(subject_level, 'filter_10_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 39  0
         1  4 27
                                          
               Accuracy : 0.9429          
                 95% CI : (0.8601, 0.9842)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 2.41e-10        
                                          
                  Kappa : 0.8826          
                                          
 Mcnemar's Test P-Value : 0.1336          
                                          
            Sensitivity : 1.0000          
            Specificity : 0.9070          
         Pos Pred Value : 0.8710          
         Neg Pred Value : 1.0000          
             Prevalence : 0.3857          
         Detection Rate : 0.3857          
   Detection Prevalence : 0.4429          
      Balanced Accuracy : 0.9535          
                                          
       'Positive' Class : 1               
                                          

Segmentations: zero all pixels < 75 and > 150

compute_aupr(subject_level$filter_75, subject_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150')

compute_auroc(subject_level$filter_75, subject_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150')

confusion_matrix(subject_level, 'filter_75')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 36  1
         1  7 26
                                          
               Accuracy : 0.8857          
                 95% CI : (0.7872, 0.9493)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 4.352e-07       
                                          
                  Kappa : 0.7684          
                                          
 Mcnemar's Test P-Value : 0.0771          
                                          
            Sensitivity : 0.9630          
            Specificity : 0.8372          
         Pos Pred Value : 0.7879          
         Neg Pred Value : 0.9730          
             Prevalence : 0.3857          
         Detection Rate : 0.3714          
   Detection Prevalence : 0.4714          
      Balanced Accuracy : 0.9001          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_75_binarized, subject_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150 and binarize')

compute_auroc(subject_level$filter_75_binarized, subject_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150 and binarize')

confusion_matrix(subject_level, 'filter_75_binarized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 33  0
         1 10 27
                                          
               Accuracy : 0.8571          
                 95% CI : (0.7529, 0.9293)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 7.748e-06       
                                          
                  Kappa : 0.718           
                                          
 Mcnemar's Test P-Value : 0.004427        
                                          
            Sensitivity : 1.0000          
            Specificity : 0.7674          
         Pos Pred Value : 0.7297          
         Neg Pred Value : 1.0000          
             Prevalence : 0.3857          
         Detection Rate : 0.3857          
   Detection Prevalence : 0.5286          
      Balanced Accuracy : 0.8837          
                                          
       'Positive' Class : 1               
                                          
compute_aupr(subject_level$filter_75_skeletonized, subject_level$label, 'PR: Segmentations: zero all pixels < 75 and > 150 and skeletonize')

compute_auroc(subject_level$filter_75_skeletonized, subject_level$label, 'ROC: Segmentations: zero all pixels < 75 and > 150 and skeletonize')

confusion_matrix(subject_level, 'filter_75_skeletonized')
Confusion Matrix and Statistics

          Reference
Prediction  0  1
         0 43 13
         1  0 14
                                          
               Accuracy : 0.8143          
                 95% CI : (0.7034, 0.8972)
    No Information Rate : 0.6143          
    P-Value [Acc > NIR] : 0.0002607       
                                          
                  Kappa : 0.5695          
                                          
 Mcnemar's Test P-Value : 0.0008741       
                                          
            Sensitivity : 0.5185          
            Specificity : 1.0000          
         Pos Pred Value : 1.0000          
         Neg Pred Value : 0.7679          
             Prevalence : 0.3857          
         Detection Rate : 0.2000          
   Detection Prevalence : 0.2000          
      Balanced Accuracy : 0.7593          
                                          
       'Positive' Class : 1               
                                          
aupr_figures <- function(predictions, labels, title, names, save_name) {
    n_black <- length(labels[labels == 1])
    n_total <- length(labels)
    for (i in seq_along(predictions)) {
        preds <- prediction(predictions[i], labels)
        aupr <- performance(preds, measure = 'aucpr')
        aupr <- aupr@y.values[[1]]
        perf <- performance(preds, 'prec', 'rec')
        if (is.null(dev.list())) {
            png(save_name,
                width=7,
                height=5,
                units='in',
                res=300)
            plot(perf,
                 main = title,
                 xlim=c(0,1),
                 ylim=c(0,1),
                 col = i,
                 lwd = 2)
        } else {
            plot(perf,
                 main = title,
                 xlim=c(0,1),
                 ylim=c(0,1),
                 col = i,
                 lwd = 2,
                 add = TRUE)
        }
    }
    abline(a = n_black/n_total,
           b = 0,
           col = 'red',
           lty = 2,
           lwd = 2)
    legend('bottomleft',
           names,
           lwd = 2,
           col = seq_along(predictions),
           bty = 'n',
           inset = c(0.1, 0.07))
    legend('bottomleft',
           paste('Null AUPR:', sprintf('%.3f', round(n_black / n_total, 3))),
           lwd = 2,
           lty = 2,
           col = 'red',
           bty = 'n',
           inset = c(0.1, 0.0))
}

Figures for Paper

aupr_figures(list(image_level$filter_0, image_level$filter_50, image_level$filter_200, image_level$filter_240),
             image_level$label,
             'Precision-Recall Curves of Thresholded RVMs',
             c('Threshold 0 AUPR:', 'Threshold 50 AUPR: ', 'Threshold 150 AUPR: ', 'Threshold 220 AUPR: '),
             './out/figures/thresholded_image.png')
aupr_figures(list(image_level$filter_0_binarized, image_level$filter_50_binarized, image_level$filter_200_binarized, image_level$filter_240_binarized),
             image_level$label,
             'Precision-Recall Curves of Binarized RVMs',
             c('Threshold 0 AUPR:', 'Threshold 50 AUPR: ', 'Threshold 150 AUPR: ', 'Threshold 220 AUPR: '),
             './out/figures/binarized_image.png')
aupr_figures(list(image_level$filter_0_skeletonized, image_level$filter_50_skeletonized, image_level$filter_200_skeletonized, image_level$filter_240_skeletonized),
             image_level$label,
             'Precision-Recall Curves of Skeletonized RVMs',
             c('Threshold 0 AUPR:', 'Threshold 50 AUPR: ', 'Threshold 150 AUPR: ', 'Threshold 220 AUPR: '),
             './out/figures/skeletonized_image.png')
aupr_figures(list(subject_level$filter_0, subject_level$filter_50, subject_level$filter_200, subject_level$filter_240),
             subject_level$label,
             'Precision-Recall Curves of Thresholded RVMs',
             c('Threshold 0 AUPR:', 'Threshold 50 AUPR: ', 'Threshold 150 AUPR: ', 'Threshold 220 AUPR: '),
             './out/figures/thresholded_subject.png')
aupr_figures(list(subject_level$filter_0_binarized, subject_level$filter_50_binarized, subject_level$filter_200_binarized, subject_level$filter_240_binarized),
             subject_level$label,
             'Precision-Recall Curves of Binarized RVMs',
             c('Threshold 0 AUPR:', 'Threshold 50 AUPR: ', 'Threshold 150 AUPR: ', 'Threshold 220 AUPR: '),
             './out/figures/binarized_subject.png')
aupr_figures(list(subject_level$filter_0_skeletonized, subject_level$filter_50_skeletonized, subject_level$filter_200_skeletonized, subject_level$filter_240_skeletonized),
             subject_level$label,
             'Precision-Recall Curves of Skeletonized RVMs',
             c('Threshold 0 AUPR:', 'Threshold 50 AUPR: ', 'Threshold 150 AUPR: ', 'Threshold 220 AUPR: '),
             './out/figures/skeletonized_subject.png')

Evaluate on Manual Segmentations

manual_binary_0 <- read_csv('./out/probabilities/manual_segmentations_binarized_0.csv') %>%
    mutate(img_name = basename(img_loc)) %>%
    separate(img_name, c('site', 'subject_id')) %>%
    filter(!is.na(as.numeric(subject_id))) %>%
    mutate(subject_id = paste(toupper(site), subject_id, sep = '-')) %>%
    select(subject_id, probability) %>%
    inner_join(irop_data) %>%
    filter(race == 'African American' | race == 'Caucasian/White') %>%
    mutate(race = if_else(race == 'African American', 1, 0),
           across(race, as.factor))

── Column specification ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  img_loc = col_character(),
  probability = col_double()
)
Warning: Expected 2 pieces. Additional pieces discarded in 196 rows [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, ...].
Warning in mask$eval_all_filter(dots, env_filter) :
  NAs introduced by coercion
Joining, by = "subject_id"
manual_skeleton_0 <- read_csv('./out/probabilities/manual_segmentations_skeletonized_0.csv') %>%
    mutate(img_name = basename(img_loc)) %>%
    separate(img_name, c('site', 'subject_id')) %>%
    filter(!is.na(as.numeric(subject_id))) %>%
    mutate(subject_id = paste(toupper(site), subject_id, sep = '-')) %>%
    select(subject_id, probability) %>%
    inner_join(irop_data) %>%
    filter(race == 'African American' | race == 'Caucasian/White') %>%
    mutate(race = if_else(race == 'African American', 1, 0),
           across(race, as.factor))

── Column specification ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  img_loc = col_character(),
  probability = col_double()
)
Warning: Expected 2 pieces. Additional pieces discarded in 196 rows [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, ...].
Warning in mask$eval_all_filter(dots, env_filter) :
  NAs introduced by coercion
Joining, by = "subject_id"
manual_binary_50 <- read_csv('./out/probabilities/manual_segmentations_binarized_50.csv') %>%
    mutate(img_name = basename(img_loc)) %>%
    separate(img_name, c('site', 'subject_id')) %>%
    filter(!is.na(as.numeric(subject_id))) %>%
    mutate(subject_id = paste(toupper(site), subject_id, sep = '-')) %>%
    select(subject_id, probability) %>%
    inner_join(irop_data) %>%
    filter(race == 'African American' | race == 'Caucasian/White') %>%
    mutate(race = if_else(race == 'African American', 1, 0),
           across(race, as.factor))

── Column specification ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  img_loc = col_character(),
  probability = col_double()
)
Warning: Expected 2 pieces. Additional pieces discarded in 196 rows [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, ...].
Warning in mask$eval_all_filter(dots, env_filter) :
  NAs introduced by coercion
Joining, by = "subject_id"
manual_skeleton_50 <- read_csv('./out/probabilities/manual_segmentations_skeletonized_50.csv') %>%
    mutate(img_name = basename(img_loc)) %>%
    separate(img_name, c('site', 'subject_id')) %>%
    filter(!is.na(as.numeric(subject_id))) %>%
    mutate(subject_id = paste(toupper(site), subject_id, sep = '-')) %>%
    select(subject_id, probability) %>%
    inner_join(irop_data) %>%
    filter(race == 'African American' | race == 'Caucasian/White') %>%
    mutate(race = if_else(race == 'African American', 1, 0),
           across(race, as.factor))

── Column specification ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  img_loc = col_character(),
  probability = col_double()
)
Warning: Expected 2 pieces. Additional pieces discarded in 196 rows [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, ...].
Warning in mask$eval_all_filter(dots, env_filter) :
  NAs introduced by coercion
Joining, by = "subject_id"
pred <- prediction(manual_binary_0$probability, manual_binary_0$race)

num_yes <- length(manual_binary_0$race[manual_binary_0$race == 1])
num_no <- length(manual_binary_0$race[manual_binary_0$race == 0])

perf <- performance(pred,'prec', 'rec')
plot(perf, colorize=TRUE, main = 'Manual Segmentations: Binarized - 0 Threshold Model', ylim = c(0,1))
abline(a = num_yes / num_no, b = 0, col = 'red', lty = 2)
aupr <- performance(pred, measure = 'aucpr')
aupr <- aupr@y.values[[1]]
text(0.8, 0.9, paste('AUPR: ', sprintf('%.3f', round(aupr, 3)), sep = ''))

pred <- prediction(manual_skeleton_0$probability, manual_skeleton_0$race)

num_yes <- length(manual_skeleton_0$race[manual_skeleton_0$race == 1])
num_no <- length(manual_skeleton_0$race[manual_skeleton_0$race == 0])

perf <- performance(pred,'prec', 'rec')
plot(perf, colorize=TRUE, main = 'Manual Segmentations: Skeletonized - 0 Threshold Model', ylim = c(0,1))
abline(a = num_yes / num_no, b = 0, col = 'red', lty = 2)
aupr <- performance(pred, measure = 'aucpr')
aupr <- aupr@y.values[[1]]
text(0.8, 0.9, paste('AUPR: ', sprintf('%.3f', round(aupr, 3)), sep = ''))

pred <- prediction(manual_binary_50$probability, manual_binary_50$race)

num_yes <- length(manual_binary_50$race[manual_binary_50$race == 1])
num_no <- length(manual_binary_50$race[manual_binary_50$race == 0])

perf <- performance(pred,'prec', 'rec')
plot(perf, colorize=TRUE, main = 'Manual Segmentations: Binarized - 50 Threshold Model', ylim = c(0,1))
abline(a = num_yes / num_no, b = 0, col = 'red', lty = 2)
aupr <- performance(pred, measure = 'aucpr')
aupr <- aupr@y.values[[1]]
text(0.8, 0.9, paste('AUPR: ', sprintf('%.3f', round(aupr, 3)), sep = ''))

pred <- prediction(manual_skeleton_50$probability, manual_skeleton_50$race)

num_yes <- length(manual_skeleton_50$race[manual_skeleton_50$race == 1])
num_no <- length(manual_skeleton_50$race[manual_skeleton_50$race == 0])

perf <- performance(pred,'prec', 'rec')
plot(perf, colorize=TRUE, main = 'Manual Segmentations: Skeletonized - 50 Threshold Model', ylim = c(0,1))
abline(a = num_yes / num_no, b = 0, col = 'red', lty = 2)
aupr <- performance(pred, measure = 'aucpr')
aupr <- aupr@y.values[[1]]
text(0.8, 0.9, paste('AUPR: ', sprintf('%.3f', round(aupr, 3)), sep = ''))

image_level <- read_csv('./out/probabilities/retcam_filtered_0.csv', col_types = cols()) %>%
    mutate(label = as.factor(if_else(str_detect(img_loc, 'black'), 1, 0)),
           image_id = as.numeric(file_path_sans_ext(basename(img_loc)))) %>%
    select(img_loc, image_id, label, retcam = probability) %>%
    bind_cols(select(read_csv('./out/probabilities/segmentations_binarized_test.csv', col_types = cols()), filter_0_binarized = probability))
pred <- prediction(image_level$filter_0_binarized, image_level$label)

num_yes <- length(image_level$label[image_level$label == 1])
num_no <- length(image_level$label[image_level$label == 0])

perf <- performance(pred,'prec', 'rec')
plot(perf, colorize=TRUE, main = 'Manual Segmentations: Skeletonized - 0 Threshold Model', ylim = c(0,1))
abline(a = num_yes / num_no, b = 0, col = 'red', lty = 2)
aupr <- performance(pred, measure = 'aucpr')
aupr <- aupr@y.values[[1]]
text(0.8, 0.9, paste('AUPR: ', sprintf('%.3f', round(aupr, 3)), sep = ''))

LS0tCnRpdGxlOiAnRXZhbHVhdGUgTW9kZWxzJwphdXRob3I6ICdBdXRob3I6IEFhcm9uIFMgQ295bmVyLCBQaEQnCmRhdGU6ICdMYXN0IHVwZGF0ZTogYHIgU3lzLkRhdGUoKWAnCm91dHB1dDoKICAgIGh0bWxfbm90ZWJvb2s6CiAgICAgICAgdG9jOiB5ZXMKICAgICAgICB0b2NfZmxvYXQ6IHllcwogICAgICAgIHRvY19kZXB0aDogMwotLS0KCgojIFNldHVwCmBgYHtyfQpsaWJyYXJ5KHRpZHl2ZXJzZSkKbGlicmFyeShqYW5pdG9yKQpsaWJyYXJ5KHRvb2xzKQpsaWJyYXJ5KGNhcmV0KQpsaWJyYXJ5KFJPQ1IpCmxpYnJhcnkoY2FyZXQpCmBgYAoKCmBgYHtyfQpjb21wdXRlX2F1cm9jIDwtIGZ1bmN0aW9uKHByZWRpY3Rpb25zLCBsYWJlbHMsIHRpdGxlKSB7CiAgICBwcmVkIDwtIHByZWRpY3Rpb24ocHJlZGljdGlvbnMsIGxhYmVscykKICAgIGF1cm9jIDwtIHBlcmZvcm1hbmNlKHByZWQsIG1lYXN1cmUgPSAnYXVjJykKICAgIGF1cm9jIDwtIGF1cm9jQHkudmFsdWVzW1sxXV0KICAgIHBlcmYgPC0gcGVyZm9ybWFuY2UocHJlZCwgJ3RwcicsICdmcHInKQogICAgcGxvdChwZXJmLAogICAgICAgICBtYWluID0gdGl0bGUsCiAgICAgICAgIHhsaW09YygwLDEpLAogICAgICAgICB5bGltPWMoMCwxKSwKICAgICAgICAgY29sb3JpemUgPSBGQUxTRSkKICAgIGFibGluZShhID0gMCwKICAgICAgICAgICBiID0gMSwKICAgICAgICAgICBjb2wgPSAncmVkJywKICAgICAgICAgICBsdHkgPSAyKQogICAgdGV4dCgwLjgsCiAgICAgICAgIDAuMjUsCiAgICAgICAgIHBhc3RlKCdBVVJPQzonLCBzcHJpbnRmKCclLjNmJywgcm91bmQoYXVyb2MsIDMpKSkpCn0KYGBgCgoKYGBge3J9CmNvbXB1dGVfYXVwciA8LSBmdW5jdGlvbihwcmVkaWN0aW9ucywgbGFiZWxzLCB0aXRsZSkgewogICAgbl9ibGFjayA8LSBsZW5ndGgobGFiZWxzW2xhYmVscyA9PSAxXSkKICAgIG5fdG90YWwgPC0gbGVuZ3RoKGxhYmVscykKICAgIHByZWQgPC0gcHJlZGljdGlvbihwcmVkaWN0aW9ucywgbGFiZWxzKQogICAgYXVwciA8LSBwZXJmb3JtYW5jZShwcmVkLCBtZWFzdXJlID0gJ2F1Y3ByJykKICAgIGF1cHIgPC0gYXVwckB5LnZhbHVlc1tbMV1dCiAgICBwZXJmIDwtIHBlcmZvcm1hbmNlKHByZWQsICdwcmVjJywgJ3JlYycpCiAgICBwbG90KHBlcmYsCiAgICAgICAgIG1haW4gPSB0aXRsZSwKICAgICAgICAgeGxpbT1jKDAsMSksCiAgICAgICAgIHlsaW09YygwLDEpLAogICAgICAgICBjb2xvcml6ZSA9IEZBTFNFKQogICAgYWJsaW5lKGEgPSBuX2JsYWNrL25fdG90YWwsCiAgICAgICAgICAgYiA9IDAsCiAgICAgICAgICAgY29sID0gJ3JlZCcsCiAgICAgICAgICAgbHR5ID0gMikKICAgIHRleHQoMC4yLAogICAgICAgICAwLjIsCiAgICAgICAgIHBhc3RlKCdOdWxsIEFVUFI6Jywgc3ByaW50ZignJS4zZicsIHJvdW5kKG5fYmxhY2sgLyBuX3RvdGFsLCAzKSkpKQogICAgdGV4dCgwLjgsCiAgICAgICAgIDAuMiwKICAgICAgICAgcGFzdGUoJ0FVUFI6Jywgc3ByaW50ZignJS4zZicsIHJvdW5kKGF1cHIsIDMpKSkpCn0KYGBgCgoKYGBge3J9CmNvbmZ1c2lvbl9tYXRyaXggPC0gZnVuY3Rpb24oZGYsIGNvbHVtbiwgdGhyZXNob2xkPTAuNSkgewogICAgcHJlZHMgPC0gYXMuZmFjdG9yKGlmX2Vsc2UoZGZbY29sdW1uXSA+PSB0aHJlc2hvbGQsIDEsIDApKQogICAgY29uZnVzaW9uTWF0cml4KHByZWRzLCBkZiRsYWJlbCwgcG9zaXRpdmUgPSAnMScpCn0KYGBgCgoKCiMgTG9hZCBkYXRhCmBgYHtyfQppcm9wX2RhdGEgPC0gcmVhZF9jc3YoJy9Wb2x1bWVzL0V4dGVybmFsL2lyb3BfZGF0YS9pcm9wXzA3MDkyMDIwLmNzdicpICU+JQogICAgY2xlYW5fbmFtZXMoKSAlPiUKICAgIGRpc3RpbmN0KHN1YmplY3RfaWQsIC5rZWVwX2FsbCA9IFRSVUUpICU+JQogICAgc2VsZWN0KHN1YmplY3RfaWQsIHJhY2UpCgp0ZXN0X2RhdGEgPC0gcmVhZF9jc3YoJy4vb3V0L2RhdGFzZXRzL3Rlc3RfZGF0YS5jc3YnKSAlPiUKICAgIHNlbGVjdChzdWJqZWN0X2lkLCBpbWFnZV9pZCkKCgppbWFnZV9sZXZlbCA8LSByZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9yZXRjYW1fZmlsdGVyZWRfMC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpICU+JQogICAgbXV0YXRlKGxhYmVsID0gYXMuZmFjdG9yKGlmX2Vsc2Uoc3RyX2RldGVjdChpbWdfbG9jLCAnYmxhY2snKSwgMSwgMCkpLAogICAgICAgICAgIGltYWdlX2lkID0gYXMubnVtZXJpYyhmaWxlX3BhdGhfc2Fuc19leHQoYmFzZW5hbWUoaW1nX2xvYykpKSkgJT4lCiAgICBzZWxlY3QoaW1nX2xvYywgaW1hZ2VfaWQsIGxhYmVsLCByZXRjYW0gPSBwcm9iYWJpbGl0eSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3JldGNhbV9maWx0ZXJlZF8wX3JhbmRvbS5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCByZXRjYW1fcmFuZG9tID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIAogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzBfcmFuZG9tLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8wX3JhbmRvbSA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMF9iaW5hcml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzBfYmluYXJpemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8wX3NrZWxldG9uaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMF9za2VsZXRvbml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfNTAuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzUwID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF81MF9iaW5hcml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzUwX2JpbmFyaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfNTBfc2tlbGV0b25pemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl81MF9za2VsZXRvbml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQoKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8xMDAuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzEwMCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMTAwX2JpbmFyaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMTAwX2JpbmFyaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMTAwX3NrZWxldG9uaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMTAwX3NrZWxldG9uaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCgogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzE1MC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMTUwID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8xNTBfYmluYXJpemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8xNTBfYmluYXJpemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8xNTBfc2tlbGV0b25pemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8xNTBfc2tlbGV0b25pemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjAwLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yMDAgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzIwMF9iaW5hcml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzIwMF9iaW5hcml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzIwMF9za2VsZXRvbml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzIwMF9za2VsZXRvbml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQoKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yMTAuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzIxMCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjEwX2JpbmFyaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjEwX2JpbmFyaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjEwX3NrZWxldG9uaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjEwX3NrZWxldG9uaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCgogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzIyMC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjIwID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yMjBfYmluYXJpemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yMjBfYmluYXJpemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yMjBfc2tlbGV0b25pemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yMjBfc2tlbGV0b25pemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjMwLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yMzAgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzIzMF9iaW5hcml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzIzMF9iaW5hcml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzIzMF9za2VsZXRvbml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzIzMF9za2VsZXRvbml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQoKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yNDAuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzI0MCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjQwX2JpbmFyaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjQwX2JpbmFyaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjQwX3NrZWxldG9uaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjQwX3NrZWxldG9uaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCgogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzI1MC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMjUwID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yNTBfYmluYXJpemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yNTBfYmluYXJpemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8yNTBfc2tlbGV0b25pemVkLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yNTBfc2tlbGV0b25pemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMjU3LmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8yNTcgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzI1N19iaW5hcml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzI1N19iaW5hcml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzI1N19za2VsZXRvbml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzI1N19za2VsZXRvbml6ZWQgPSBwcm9iYWJpbGl0eSkpICU+JQoKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19maWx0ZXJlZF8wXzEwLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSksIGZpbHRlcl8xMCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMF8xMF9iaW5hcml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzEwX2JpbmFyaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfMF8xMF9za2VsZXRvbml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzEwX3NrZWxldG9uaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCgogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzc1XzE1MC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfNzUgPSBwcm9iYWJpbGl0eSkpICU+JQogICAgYmluZF9jb2xzKHNlbGVjdChyZWFkX2NzdignLi9vdXQvcHJvYmFiaWxpdGllcy9zZWdtZW50YXRpb25zX2ZpbHRlcmVkXzc1XzE1MF9iaW5hcml6ZWQuY3N2JywgY29sX3R5cGVzID0gY29scygpKSwgZmlsdGVyXzc1X2JpbmFyaXplZCA9IHByb2JhYmlsaXR5KSkgJT4lCiAgICBiaW5kX2NvbHMoc2VsZWN0KHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3NlZ21lbnRhdGlvbnNfZmlsdGVyZWRfNzVfMTUwX3NrZWxldG9uaXplZC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfNzVfc2tlbGV0b25pemVkID0gcHJvYmFiaWxpdHkpKSAlPiUKICAgIAogICAgaW5uZXJfam9pbih0ZXN0X2RhdGEsIGJ5ID0gJ2ltYWdlX2lkJykgJT4lCiAgICBzZWxlY3Qoc3ViamVjdF9pZCwgZXZlcnl0aGluZygpLCAtaW1nX2xvYykKCgoKc3ViamVjdF9sZXZlbCA8LSBpbWFnZV9sZXZlbCAlPiUKICAgIGdyb3VwX2J5KHN1YmplY3RfaWQpICU+JQogICAgbXV0YXRlKGFjcm9zcyhjKC1pbWFnZV9pZCwgLWxhYmVsKSwgbWVkaWFuKSkgJT4lCiAgICB1bmdyb3VwKCkgJT4lCiAgICBkaXN0aW5jdChzdWJqZWN0X2lkLCAua2VlcF9hbGwgPSBUUlVFKQpgYGAKCgojIEltYWdlLWxldmVsIEFuYWx5c2lzCgojIyMgUmV0Q2FtIEltYWdlcwoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRyZXRjYW0sIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFJhdyBSZXRDYW0gSW1hZ2VzJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRyZXRjYW0sIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBSYXcgUmV0Q2FtIEltYWdlcycpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdyZXRjYW0nKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkcmV0Y2FtX3JhbmRvbSwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogUmF3IFJldENhbSBJbWFnZXMgLSBTaHVmZmxlZCBMYWJlbHMnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJHJldGNhbV9yYW5kb20sIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBSYXcgUmV0Q2FtIEltYWdlcyAtIFNodWZmbGVkIExhYmVscycpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdyZXRjYW1fcmFuZG9tJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8wLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl8wX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8wX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8wX3NrZWxldG9uaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMF9yYW5kb20sIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAgLSBTaHVmZmxlZCBMYWJlbHMnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8wX3JhbmRvbSwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAgLSBTaHVmZmxlZCBMYWJlbHMnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzBfcmFuZG9tJykKYGBgCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl81MCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNTAnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl81MCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl81MCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfNTBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA1MCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl81MF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA1MCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KGltYWdlX2xldmVsLCAnZmlsdGVyXzBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl81MF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzUwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfNTBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMDAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMTAwLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8xMDAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwMF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwMCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8xMDBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMTAwX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMTAwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMDAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8xMDBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxNTAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzE1MCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMTUwLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8xNTAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzE1MF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDE1MCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8xNTBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMTUwX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMTUwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzE1MF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxNTAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8xNTBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMDAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIwMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjAwLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yMDAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIwMF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIwMCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yMDBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjAwX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMjAwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIwMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMDAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yMDBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMTAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIxMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjEwLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yMTAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIxMF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIxMCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yMTBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjEwX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMjEwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIxMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMTAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yMTBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMjAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIyMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjIwLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yMjAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIyMF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIyMCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yMjBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjIwX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMjIwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIyMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMjAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yMjBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMzAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIzMCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjMwLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yMzAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIzMF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIzMCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yMzBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjMwX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMjMwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzIzMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMzAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yMzBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNDAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI0MCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjQwLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yNDAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI0MF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI0MCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yNDBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjQwX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMjQwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI0MF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNDAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yNDBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI1MCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjUwLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yNTAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI1MF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1MCBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yNTBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjUwX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMjUwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI1MF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yNTBfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTcKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI1NywgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3JykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMjU3LCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3JykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yNTcnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI1N19iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1NyBhbmQgYmluYXJpemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl8yNTdfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3IGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMjU3X2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMjU3X3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3IGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzI1N19za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTcgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8yNTdfc2tlbGV0b25pemVkJykKYGBgCgoKCiMjIyBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPiAxMAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfMTAsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMTAsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMCcpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfMTAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMTBfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8xMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoaW1hZ2VfbGV2ZWwkZmlsdGVyXzEwX3NrZWxldG9uaXplZCwgaW1hZ2VfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfMTBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl8xMF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDc1IGFuZCA+IDE1MAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfNzUsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDc1IGFuZCA+IDE1MCcpCmNvbXB1dGVfYXVyb2MoaW1hZ2VfbGV2ZWwkZmlsdGVyXzc1LCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNzUgYW5kID4gMTUwJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl83NScpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihpbWFnZV9sZXZlbCRmaWx0ZXJfNzVfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA3NSBhbmQgPiAxNTAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhpbWFnZV9sZXZlbCRmaWx0ZXJfNzVfYmluYXJpemVkLCBpbWFnZV9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNzUgYW5kID4gMTUwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoaW1hZ2VfbGV2ZWwsICdmaWx0ZXJfNzVfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKGltYWdlX2xldmVsJGZpbHRlcl83NV9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDc1IGFuZCA+IDE1MCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKGltYWdlX2xldmVsJGZpbHRlcl83NV9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA3NSBhbmQgPiAxNTAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChpbWFnZV9sZXZlbCwgJ2ZpbHRlcl83NV9za2VsZXRvbml6ZWQnKQpgYGAKCgoKCgoKCgoKCgojIFN1YmplY3QtbGV2ZWwgQW5hbHlzaXMKCiMjIyBSZXRDYW0gSW1hZ2VzCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkcmV0Y2FtLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFJhdyBSZXRDYW0gSW1hZ2VzJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJHJldGNhbSwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogUmF3IFJldENhbSBJbWFnZXMnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdyZXRjYW0nKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRyZXRjYW1fcmFuZG9tLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFJhdyBSZXRDYW0gSW1hZ2VzIC0gU2h1ZmZsZWQgTGFiZWxzJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJHJldGNhbV9yYW5kb20sIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFJhdyBSZXRDYW0gSW1hZ2VzIC0gU2h1ZmZsZWQgTGFiZWxzJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAncmV0Y2FtX3JhbmRvbScpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8wLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAnKQpjb21wdXRlX2F1cm9jKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8wX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8wX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMCBhbmQgc2tlbGV0b25pemUnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMF9za2VsZXRvbml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMF9yYW5kb20sIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMCAtIFNodWZmbGVkIExhYmVscycpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMF9yYW5kb20sIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDAgLSBTaHVmZmxlZCBMYWJlbHMnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMF9yYW5kb20nKQpgYGAKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNTAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfNTAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNTAnKQpjb21wdXRlX2F1cm9jKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzUwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA1MCcpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl81MCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl81MF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNTAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl81MF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwIGFuZCBiaW5hcml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8wX2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl81MF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNTAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl81MF9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDUwIGFuZCBza2VsZXRvbml6ZScpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl81MF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwMAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8xMDAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8xMDAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwMCcpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8xMDAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTAwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMDAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8xMDBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMDAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzEwMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTAwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMDAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8xMDBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMDAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzEwMF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDE1MAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8xNTAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTUwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8xNTAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDE1MCcpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8xNTAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTUwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxNTAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8xNTBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxNTAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzE1MF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTUwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxNTAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8xNTBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxNTAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzE1MF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIwMAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8yMDAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjAwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMDAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIwMCcpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMDAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjAwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMDAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMDBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMDAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzIwMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjAwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMDAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMDBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMDAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzIwMF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIxMAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8yMTAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjEwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMTAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIxMCcpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMTAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjEwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMTAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMTBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMTAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzIxMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjEwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMTAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMTBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMTAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzIxMF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIyMAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8yMjAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjIwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMjAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIyMCcpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMjAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjIwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMjAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMjBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMjAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzIyMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjIwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMjAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMjBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMjAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzIyMF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIzMAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8yMzAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjMwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMzAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDIzMCcpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yMzAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjMwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMzAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMzBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMzAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzIzMF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjMwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMzAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yMzBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyMzAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzIzMF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI0MAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8yNDAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjQwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yNDAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI0MCcpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yNDAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjQwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNDAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yNDBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNDAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzI0MF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjQwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNDAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yNDBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNDAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzI0MF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1MAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8yNTAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjUwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yNTAsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1MCcpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yNTAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjUwX2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTAgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yNTBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzI1MF9iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjUwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTAgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yNTBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzI1MF9za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1NwoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl8yNTcsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMjU3JykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yNTcsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDI1NycpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl8yNTcnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjU3X2JpbmFyaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTcgYW5kIGJpbmFyaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yNTdfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTcgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzI1N19iaW5hcml6ZWQnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjU3X3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTcgYW5kIHNrZWxldG9uaXplJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8yNTdfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAyNTcgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzI1N19za2VsZXRvbml6ZWQnKQpgYGAKCgoKIyMjIFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA+IDEwCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzEwLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwJykKY29tcHV0ZV9hdXJvYyhzdWJqZWN0X2xldmVsJGZpbHRlcl8xMCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMTAnKQpgYGAKCmBgYHtyfQpjb21wdXRlX2F1cHIoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDEwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfMTBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMCBhbmQgYmluYXJpemUnKQpjb25mdXNpb25fbWF0cml4KHN1YmplY3RfbGV2ZWwsICdmaWx0ZXJfMTBfYmluYXJpemVkJykKYGBgCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzEwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1BSOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCAxMCBhbmQgc2tlbGV0b25pemUnKQpjb21wdXRlX2F1cm9jKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzEwX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRsYWJlbCwgJ1JPQzogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgMTAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzEwX3NrZWxldG9uaXplZCcpCmBgYAoKCgojIyMgU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNzUgYW5kID4gMTUwCgpgYGB7cn0KY29tcHV0ZV9hdXByKHN1YmplY3RfbGV2ZWwkZmlsdGVyXzc1LCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUFI6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDc1IGFuZCA+IDE1MCcpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfNzUsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdST0M6IFNlZ21lbnRhdGlvbnM6IHplcm8gYWxsIHBpeGVscyA8IDc1IGFuZCA+IDE1MCcpCmNvbmZ1c2lvbl9tYXRyaXgoc3ViamVjdF9sZXZlbCwgJ2ZpbHRlcl83NScpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl83NV9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNzUgYW5kID4gMTUwIGFuZCBiaW5hcml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfNzVfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA3NSBhbmQgPiAxNTAgYW5kIGJpbmFyaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzc1X2JpbmFyaXplZCcpCmBgYAoKYGBge3J9CmNvbXB1dGVfYXVwcihzdWJqZWN0X2xldmVsJGZpbHRlcl83NV9za2VsZXRvbml6ZWQsIHN1YmplY3RfbGV2ZWwkbGFiZWwsICdQUjogU2VnbWVudGF0aW9uczogemVybyBhbGwgcGl4ZWxzIDwgNzUgYW5kID4gMTUwIGFuZCBza2VsZXRvbml6ZScpCmNvbXB1dGVfYXVyb2Moc3ViamVjdF9sZXZlbCRmaWx0ZXJfNzVfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGxhYmVsLCAnUk9DOiBTZWdtZW50YXRpb25zOiB6ZXJvIGFsbCBwaXhlbHMgPCA3NSBhbmQgPiAxNTAgYW5kIHNrZWxldG9uaXplJykKY29uZnVzaW9uX21hdHJpeChzdWJqZWN0X2xldmVsLCAnZmlsdGVyXzc1X3NrZWxldG9uaXplZCcpCmBgYAoKCgpgYGB7cn0KYXVwcl9maWd1cmVzIDwtIGZ1bmN0aW9uKHByZWRpY3Rpb25zLCBsYWJlbHMsIHRpdGxlLCBuYW1lcywgc2F2ZV9uYW1lKSB7CiAgICBuX2JsYWNrIDwtIGxlbmd0aChsYWJlbHNbbGFiZWxzID09IDFdKQogICAgbl90b3RhbCA8LSBsZW5ndGgobGFiZWxzKQogICAgZm9yIChpIGluIHNlcV9hbG9uZyhwcmVkaWN0aW9ucykpIHsKICAgICAgICBwcmVkcyA8LSBwcmVkaWN0aW9uKHByZWRpY3Rpb25zW2ldLCBsYWJlbHMpCiAgICAgICAgYXVwciA8LSBwZXJmb3JtYW5jZShwcmVkcywgbWVhc3VyZSA9ICdhdWNwcicpCiAgICAgICAgYXVwciA8LSBhdXByQHkudmFsdWVzW1sxXV0KICAgICAgICBwZXJmIDwtIHBlcmZvcm1hbmNlKHByZWRzLCAncHJlYycsICdyZWMnKQogICAgICAgIGlmIChpcy5udWxsKGRldi5saXN0KCkpKSB7CiAgICAgICAgICAgIHBuZyhzYXZlX25hbWUsCiAgICAgICAgICAgICAgICB3aWR0aD03LAogICAgICAgICAgICAgICAgaGVpZ2h0PTUsCiAgICAgICAgICAgICAgICB1bml0cz0naW4nLAogICAgICAgICAgICAgICAgcmVzPTMwMCkKICAgICAgICAgICAgcGxvdChwZXJmLAogICAgICAgICAgICAgICAgIG1haW4gPSB0aXRsZSwKICAgICAgICAgICAgICAgICB4bGltPWMoMCwxKSwKICAgICAgICAgICAgICAgICB5bGltPWMoMCwxKSwKICAgICAgICAgICAgICAgICBjb2wgPSBpLAogICAgICAgICAgICAgICAgIGx3ZCA9IDIpCiAgICAgICAgfSBlbHNlIHsKICAgICAgICAgICAgcGxvdChwZXJmLAogICAgICAgICAgICAgICAgIG1haW4gPSB0aXRsZSwKICAgICAgICAgICAgICAgICB4bGltPWMoMCwxKSwKICAgICAgICAgICAgICAgICB5bGltPWMoMCwxKSwKICAgICAgICAgICAgICAgICBjb2wgPSBpLAogICAgICAgICAgICAgICAgIGx3ZCA9IDIsCiAgICAgICAgICAgICAgICAgYWRkID0gVFJVRSkKICAgICAgICB9CiAgICB9CiAgICBhYmxpbmUoYSA9IG5fYmxhY2svbl90b3RhbCwKICAgICAgICAgICBiID0gMCwKICAgICAgICAgICBjb2wgPSAncmVkJywKICAgICAgICAgICBsdHkgPSAyLAogICAgICAgICAgIGx3ZCA9IDIpCiAgICBsZWdlbmQoJ2JvdHRvbWxlZnQnLAogICAgICAgICAgIG5hbWVzLAogICAgICAgICAgIGx3ZCA9IDIsCiAgICAgICAgICAgY29sID0gc2VxX2Fsb25nKHByZWRpY3Rpb25zKSwKICAgICAgICAgICBidHkgPSAnbicsCiAgICAgICAgICAgaW5zZXQgPSBjKDAuMSwgMC4wNykpCiAgICBsZWdlbmQoJ2JvdHRvbWxlZnQnLAogICAgICAgICAgIHBhc3RlKCdOdWxsIEFVUFI6Jywgc3ByaW50ZignJS4zZicsIHJvdW5kKG5fYmxhY2sgLyBuX3RvdGFsLCAzKSkpLAogICAgICAgICAgIGx3ZCA9IDIsCiAgICAgICAgICAgbHR5ID0gMiwKICAgICAgICAgICBjb2wgPSAncmVkJywKICAgICAgICAgICBidHkgPSAnbicsCiAgICAgICAgICAgaW5zZXQgPSBjKDAuMSwgMC4wKSkKfQpgYGAKCgojIyBGaWd1cmVzIGZvciBQYXBlcgpgYGB7cn0KYXVwcl9maWd1cmVzKGxpc3QoaW1hZ2VfbGV2ZWwkZmlsdGVyXzAsIGltYWdlX2xldmVsJGZpbHRlcl81MCwgaW1hZ2VfbGV2ZWwkZmlsdGVyXzIwMCwgaW1hZ2VfbGV2ZWwkZmlsdGVyXzI0MCksCiAgICAgICAgICAgICBpbWFnZV9sZXZlbCRsYWJlbCwKICAgICAgICAgICAgICdQcmVjaXNpb24tUmVjYWxsIEN1cnZlcyBvZiBUaHJlc2hvbGRlZCBSVk1zJywKICAgICAgICAgICAgIGMoJ1RocmVzaG9sZCAwIEFVUFI6JywgJ1RocmVzaG9sZCA1MCBBVVBSOiAnLCAnVGhyZXNob2xkIDE1MCBBVVBSOiAnLCAnVGhyZXNob2xkIDIyMCBBVVBSOiAnKSwKICAgICAgICAgICAgICcuL291dC9maWd1cmVzL3RocmVzaG9sZGVkX2ltYWdlLnBuZycpCmBgYAoKYGBge3J9CmF1cHJfZmlndXJlcyhsaXN0KGltYWdlX2xldmVsJGZpbHRlcl8wX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkZmlsdGVyXzUwX2JpbmFyaXplZCwgaW1hZ2VfbGV2ZWwkZmlsdGVyXzIwMF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGZpbHRlcl8yNDBfYmluYXJpemVkKSwKICAgICAgICAgICAgIGltYWdlX2xldmVsJGxhYmVsLAogICAgICAgICAgICAgJ1ByZWNpc2lvbi1SZWNhbGwgQ3VydmVzIG9mIEJpbmFyaXplZCBSVk1zJywKICAgICAgICAgICAgIGMoJ1RocmVzaG9sZCAwIEFVUFI6JywgJ1RocmVzaG9sZCA1MCBBVVBSOiAnLCAnVGhyZXNob2xkIDE1MCBBVVBSOiAnLCAnVGhyZXNob2xkIDIyMCBBVVBSOiAnKSwKICAgICAgICAgICAgICcuL291dC9maWd1cmVzL2JpbmFyaXplZF9pbWFnZS5wbmcnKQpgYGAKCmBgYHtyfQphdXByX2ZpZ3VyZXMobGlzdChpbWFnZV9sZXZlbCRmaWx0ZXJfMF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGZpbHRlcl81MF9za2VsZXRvbml6ZWQsIGltYWdlX2xldmVsJGZpbHRlcl8yMDBfc2tlbGV0b25pemVkLCBpbWFnZV9sZXZlbCRmaWx0ZXJfMjQwX3NrZWxldG9uaXplZCksCiAgICAgICAgICAgICBpbWFnZV9sZXZlbCRsYWJlbCwKICAgICAgICAgICAgICdQcmVjaXNpb24tUmVjYWxsIEN1cnZlcyBvZiBTa2VsZXRvbml6ZWQgUlZNcycsCiAgICAgICAgICAgICBjKCdUaHJlc2hvbGQgMCBBVVBSOicsICdUaHJlc2hvbGQgNTAgQVVQUjogJywgJ1RocmVzaG9sZCAxNTAgQVVQUjogJywgJ1RocmVzaG9sZCAyMjAgQVVQUjogJyksCiAgICAgICAgICAgICAnLi9vdXQvZmlndXJlcy9za2VsZXRvbml6ZWRfaW1hZ2UucG5nJykKYGBgCgpgYGB7cn0KYXVwcl9maWd1cmVzKGxpc3Qoc3ViamVjdF9sZXZlbCRmaWx0ZXJfMCwgc3ViamVjdF9sZXZlbCRmaWx0ZXJfNTAsIHN1YmplY3RfbGV2ZWwkZmlsdGVyXzIwMCwgc3ViamVjdF9sZXZlbCRmaWx0ZXJfMjQwKSwKICAgICAgICAgICAgIHN1YmplY3RfbGV2ZWwkbGFiZWwsCiAgICAgICAgICAgICAnUHJlY2lzaW9uLVJlY2FsbCBDdXJ2ZXMgb2YgVGhyZXNob2xkZWQgUlZNcycsCiAgICAgICAgICAgICBjKCdUaHJlc2hvbGQgMCBBVVBSOicsICdUaHJlc2hvbGQgNTAgQVVQUjogJywgJ1RocmVzaG9sZCAxNTAgQVVQUjogJywgJ1RocmVzaG9sZCAyMjAgQVVQUjogJyksCiAgICAgICAgICAgICAnLi9vdXQvZmlndXJlcy90aHJlc2hvbGRlZF9zdWJqZWN0LnBuZycpCmBgYAoKYGBge3J9CmF1cHJfZmlndXJlcyhsaXN0KHN1YmplY3RfbGV2ZWwkZmlsdGVyXzBfYmluYXJpemVkLCBzdWJqZWN0X2xldmVsJGZpbHRlcl81MF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkZmlsdGVyXzIwMF9iaW5hcml6ZWQsIHN1YmplY3RfbGV2ZWwkZmlsdGVyXzI0MF9iaW5hcml6ZWQpLAogICAgICAgICAgICAgc3ViamVjdF9sZXZlbCRsYWJlbCwKICAgICAgICAgICAgICdQcmVjaXNpb24tUmVjYWxsIEN1cnZlcyBvZiBCaW5hcml6ZWQgUlZNcycsCiAgICAgICAgICAgICBjKCdUaHJlc2hvbGQgMCBBVVBSOicsICdUaHJlc2hvbGQgNTAgQVVQUjogJywgJ1RocmVzaG9sZCAxNTAgQVVQUjogJywgJ1RocmVzaG9sZCAyMjAgQVVQUjogJyksCiAgICAgICAgICAgICAnLi9vdXQvZmlndXJlcy9iaW5hcml6ZWRfc3ViamVjdC5wbmcnKQpgYGAKCmBgYHtyfQphdXByX2ZpZ3VyZXMobGlzdChzdWJqZWN0X2xldmVsJGZpbHRlcl8wX3NrZWxldG9uaXplZCwgc3ViamVjdF9sZXZlbCRmaWx0ZXJfNTBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGZpbHRlcl8yMDBfc2tlbGV0b25pemVkLCBzdWJqZWN0X2xldmVsJGZpbHRlcl8yNDBfc2tlbGV0b25pemVkKSwKICAgICAgICAgICAgIHN1YmplY3RfbGV2ZWwkbGFiZWwsCiAgICAgICAgICAgICAnUHJlY2lzaW9uLVJlY2FsbCBDdXJ2ZXMgb2YgU2tlbGV0b25pemVkIFJWTXMnLAogICAgICAgICAgICAgYygnVGhyZXNob2xkIDAgQVVQUjonLCAnVGhyZXNob2xkIDUwIEFVUFI6ICcsICdUaHJlc2hvbGQgMTUwIEFVUFI6ICcsICdUaHJlc2hvbGQgMjIwIEFVUFI6ICcpLAogICAgICAgICAgICAgJy4vb3V0L2ZpZ3VyZXMvc2tlbGV0b25pemVkX3N1YmplY3QucG5nJykKYGBgCgoKCiMjIEV2YWx1YXRlIG9uIE1hbnVhbCBTZWdtZW50YXRpb25zCmBgYHtyfQptYW51YWxfYmluYXJ5XzAgPC0gcmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvbWFudWFsX3NlZ21lbnRhdGlvbnNfYmluYXJpemVkXzAuY3N2JykgJT4lCiAgICBtdXRhdGUoaW1nX25hbWUgPSBiYXNlbmFtZShpbWdfbG9jKSkgJT4lCiAgICBzZXBhcmF0ZShpbWdfbmFtZSwgYygnc2l0ZScsICdzdWJqZWN0X2lkJykpICU+JQogICAgZmlsdGVyKCFpcy5uYShhcy5udW1lcmljKHN1YmplY3RfaWQpKSkgJT4lCiAgICBtdXRhdGUoc3ViamVjdF9pZCA9IHBhc3RlKHRvdXBwZXIoc2l0ZSksIHN1YmplY3RfaWQsIHNlcCA9ICctJykpICU+JQogICAgc2VsZWN0KHN1YmplY3RfaWQsIHByb2JhYmlsaXR5KSAlPiUKICAgIGlubmVyX2pvaW4oaXJvcF9kYXRhKSAlPiUKICAgIGZpbHRlcihyYWNlID09ICdBZnJpY2FuIEFtZXJpY2FuJyB8IHJhY2UgPT0gJ0NhdWNhc2lhbi9XaGl0ZScpICU+JQogICAgbXV0YXRlKHJhY2UgPSBpZl9lbHNlKHJhY2UgPT0gJ0FmcmljYW4gQW1lcmljYW4nLCAxLCAwKSwKICAgICAgICAgICBhY3Jvc3MocmFjZSwgYXMuZmFjdG9yKSkKCm1hbnVhbF9za2VsZXRvbl8wIDwtIHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL21hbnVhbF9zZWdtZW50YXRpb25zX3NrZWxldG9uaXplZF8wLmNzdicpICU+JQogICAgbXV0YXRlKGltZ19uYW1lID0gYmFzZW5hbWUoaW1nX2xvYykpICU+JQogICAgc2VwYXJhdGUoaW1nX25hbWUsIGMoJ3NpdGUnLCAnc3ViamVjdF9pZCcpKSAlPiUKICAgIGZpbHRlcighaXMubmEoYXMubnVtZXJpYyhzdWJqZWN0X2lkKSkpICU+JQogICAgbXV0YXRlKHN1YmplY3RfaWQgPSBwYXN0ZSh0b3VwcGVyKHNpdGUpLCBzdWJqZWN0X2lkLCBzZXAgPSAnLScpKSAlPiUKICAgIHNlbGVjdChzdWJqZWN0X2lkLCBwcm9iYWJpbGl0eSkgJT4lCiAgICBpbm5lcl9qb2luKGlyb3BfZGF0YSkgJT4lCiAgICBmaWx0ZXIocmFjZSA9PSAnQWZyaWNhbiBBbWVyaWNhbicgfCByYWNlID09ICdDYXVjYXNpYW4vV2hpdGUnKSAlPiUKICAgIG11dGF0ZShyYWNlID0gaWZfZWxzZShyYWNlID09ICdBZnJpY2FuIEFtZXJpY2FuJywgMSwgMCksCiAgICAgICAgICAgYWNyb3NzKHJhY2UsIGFzLmZhY3RvcikpCgptYW51YWxfYmluYXJ5XzUwIDwtIHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL21hbnVhbF9zZWdtZW50YXRpb25zX2JpbmFyaXplZF81MC5jc3YnKSAlPiUKICAgIG11dGF0ZShpbWdfbmFtZSA9IGJhc2VuYW1lKGltZ19sb2MpKSAlPiUKICAgIHNlcGFyYXRlKGltZ19uYW1lLCBjKCdzaXRlJywgJ3N1YmplY3RfaWQnKSkgJT4lCiAgICBmaWx0ZXIoIWlzLm5hKGFzLm51bWVyaWMoc3ViamVjdF9pZCkpKSAlPiUKICAgIG11dGF0ZShzdWJqZWN0X2lkID0gcGFzdGUodG91cHBlcihzaXRlKSwgc3ViamVjdF9pZCwgc2VwID0gJy0nKSkgJT4lCiAgICBzZWxlY3Qoc3ViamVjdF9pZCwgcHJvYmFiaWxpdHkpICU+JQogICAgaW5uZXJfam9pbihpcm9wX2RhdGEpICU+JQogICAgZmlsdGVyKHJhY2UgPT0gJ0FmcmljYW4gQW1lcmljYW4nIHwgcmFjZSA9PSAnQ2F1Y2FzaWFuL1doaXRlJykgJT4lCiAgICBtdXRhdGUocmFjZSA9IGlmX2Vsc2UocmFjZSA9PSAnQWZyaWNhbiBBbWVyaWNhbicsIDEsIDApLAogICAgICAgICAgIGFjcm9zcyhyYWNlLCBhcy5mYWN0b3IpKQoKbWFudWFsX3NrZWxldG9uXzUwIDwtIHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL21hbnVhbF9zZWdtZW50YXRpb25zX3NrZWxldG9uaXplZF81MC5jc3YnKSAlPiUKICAgIG11dGF0ZShpbWdfbmFtZSA9IGJhc2VuYW1lKGltZ19sb2MpKSAlPiUKICAgIHNlcGFyYXRlKGltZ19uYW1lLCBjKCdzaXRlJywgJ3N1YmplY3RfaWQnKSkgJT4lCiAgICBmaWx0ZXIoIWlzLm5hKGFzLm51bWVyaWMoc3ViamVjdF9pZCkpKSAlPiUKICAgIG11dGF0ZShzdWJqZWN0X2lkID0gcGFzdGUodG91cHBlcihzaXRlKSwgc3ViamVjdF9pZCwgc2VwID0gJy0nKSkgJT4lCiAgICBzZWxlY3Qoc3ViamVjdF9pZCwgcHJvYmFiaWxpdHkpICU+JQogICAgaW5uZXJfam9pbihpcm9wX2RhdGEpICU+JQogICAgZmlsdGVyKHJhY2UgPT0gJ0FmcmljYW4gQW1lcmljYW4nIHwgcmFjZSA9PSAnQ2F1Y2FzaWFuL1doaXRlJykgJT4lCiAgICBtdXRhdGUocmFjZSA9IGlmX2Vsc2UocmFjZSA9PSAnQWZyaWNhbiBBbWVyaWNhbicsIDEsIDApLAogICAgICAgICAgIGFjcm9zcyhyYWNlLCBhcy5mYWN0b3IpKQpgYGAKCgpgYGB7cn0KcHJlZCA8LSBwcmVkaWN0aW9uKG1hbnVhbF9iaW5hcnlfMCRwcm9iYWJpbGl0eSwgbWFudWFsX2JpbmFyeV8wJHJhY2UpCgpudW1feWVzIDwtIGxlbmd0aChtYW51YWxfYmluYXJ5XzAkcmFjZVttYW51YWxfYmluYXJ5XzAkcmFjZSA9PSAxXSkKbnVtX25vIDwtIGxlbmd0aChtYW51YWxfYmluYXJ5XzAkcmFjZVttYW51YWxfYmluYXJ5XzAkcmFjZSA9PSAwXSkKCnBlcmYgPC0gcGVyZm9ybWFuY2UocHJlZCwncHJlYycsICdyZWMnKQpwbG90KHBlcmYsIGNvbG9yaXplPVRSVUUsIG1haW4gPSAnTWFudWFsIFNlZ21lbnRhdGlvbnM6IEJpbmFyaXplZCAtIDAgVGhyZXNob2xkIE1vZGVsJywgeWxpbSA9IGMoMCwxKSkKYWJsaW5lKGEgPSBudW1feWVzIC8gbnVtX25vLCBiID0gMCwgY29sID0gJ3JlZCcsIGx0eSA9IDIpCmF1cHIgPC0gcGVyZm9ybWFuY2UocHJlZCwgbWVhc3VyZSA9ICdhdWNwcicpCmF1cHIgPC0gYXVwckB5LnZhbHVlc1tbMV1dCnRleHQoMC44LCAwLjksIHBhc3RlKCdBVVBSOiAnLCBzcHJpbnRmKCclLjNmJywgcm91bmQoYXVwciwgMykpLCBzZXAgPSAnJykpCmBgYAoKCmBgYHtyfQpwcmVkIDwtIHByZWRpY3Rpb24obWFudWFsX3NrZWxldG9uXzAkcHJvYmFiaWxpdHksIG1hbnVhbF9za2VsZXRvbl8wJHJhY2UpCgpudW1feWVzIDwtIGxlbmd0aChtYW51YWxfc2tlbGV0b25fMCRyYWNlW21hbnVhbF9za2VsZXRvbl8wJHJhY2UgPT0gMV0pCm51bV9ubyA8LSBsZW5ndGgobWFudWFsX3NrZWxldG9uXzAkcmFjZVttYW51YWxfc2tlbGV0b25fMCRyYWNlID09IDBdKQoKcGVyZiA8LSBwZXJmb3JtYW5jZShwcmVkLCdwcmVjJywgJ3JlYycpCnBsb3QocGVyZiwgY29sb3JpemU9VFJVRSwgbWFpbiA9ICdNYW51YWwgU2VnbWVudGF0aW9uczogU2tlbGV0b25pemVkIC0gMCBUaHJlc2hvbGQgTW9kZWwnLCB5bGltID0gYygwLDEpKQphYmxpbmUoYSA9IG51bV95ZXMgLyBudW1fbm8sIGIgPSAwLCBjb2wgPSAncmVkJywgbHR5ID0gMikKYXVwciA8LSBwZXJmb3JtYW5jZShwcmVkLCBtZWFzdXJlID0gJ2F1Y3ByJykKYXVwciA8LSBhdXByQHkudmFsdWVzW1sxXV0KdGV4dCgwLjgsIDAuOSwgcGFzdGUoJ0FVUFI6ICcsIHNwcmludGYoJyUuM2YnLCByb3VuZChhdXByLCAzKSksIHNlcCA9ICcnKSkKYGBgCgoKYGBge3J9CnByZWQgPC0gcHJlZGljdGlvbihtYW51YWxfYmluYXJ5XzUwJHByb2JhYmlsaXR5LCBtYW51YWxfYmluYXJ5XzUwJHJhY2UpCgpudW1feWVzIDwtIGxlbmd0aChtYW51YWxfYmluYXJ5XzUwJHJhY2VbbWFudWFsX2JpbmFyeV81MCRyYWNlID09IDFdKQpudW1fbm8gPC0gbGVuZ3RoKG1hbnVhbF9iaW5hcnlfNTAkcmFjZVttYW51YWxfYmluYXJ5XzUwJHJhY2UgPT0gMF0pCgpwZXJmIDwtIHBlcmZvcm1hbmNlKHByZWQsJ3ByZWMnLCAncmVjJykKcGxvdChwZXJmLCBjb2xvcml6ZT1UUlVFLCBtYWluID0gJ01hbnVhbCBTZWdtZW50YXRpb25zOiBCaW5hcml6ZWQgLSA1MCBUaHJlc2hvbGQgTW9kZWwnLCB5bGltID0gYygwLDEpKQphYmxpbmUoYSA9IG51bV95ZXMgLyBudW1fbm8sIGIgPSAwLCBjb2wgPSAncmVkJywgbHR5ID0gMikKYXVwciA8LSBwZXJmb3JtYW5jZShwcmVkLCBtZWFzdXJlID0gJ2F1Y3ByJykKYXVwciA8LSBhdXByQHkudmFsdWVzW1sxXV0KdGV4dCgwLjgsIDAuOSwgcGFzdGUoJ0FVUFI6ICcsIHNwcmludGYoJyUuM2YnLCByb3VuZChhdXByLCAzKSksIHNlcCA9ICcnKSkKYGBgCgoKYGBge3J9CnByZWQgPC0gcHJlZGljdGlvbihtYW51YWxfc2tlbGV0b25fNTAkcHJvYmFiaWxpdHksIG1hbnVhbF9za2VsZXRvbl81MCRyYWNlKQoKbnVtX3llcyA8LSBsZW5ndGgobWFudWFsX3NrZWxldG9uXzUwJHJhY2VbbWFudWFsX3NrZWxldG9uXzUwJHJhY2UgPT0gMV0pCm51bV9ubyA8LSBsZW5ndGgobWFudWFsX3NrZWxldG9uXzUwJHJhY2VbbWFudWFsX3NrZWxldG9uXzUwJHJhY2UgPT0gMF0pCgpwZXJmIDwtIHBlcmZvcm1hbmNlKHByZWQsJ3ByZWMnLCAncmVjJykKcGxvdChwZXJmLCBjb2xvcml6ZT1UUlVFLCBtYWluID0gJ01hbnVhbCBTZWdtZW50YXRpb25zOiBTa2VsZXRvbml6ZWQgLSA1MCBUaHJlc2hvbGQgTW9kZWwnLCB5bGltID0gYygwLDEpKQphYmxpbmUoYSA9IG51bV95ZXMgLyBudW1fbm8sIGIgPSAwLCBjb2wgPSAncmVkJywgbHR5ID0gMikKYXVwciA8LSBwZXJmb3JtYW5jZShwcmVkLCBtZWFzdXJlID0gJ2F1Y3ByJykKYXVwciA8LSBhdXByQHkudmFsdWVzW1sxXV0KdGV4dCgwLjgsIDAuOSwgcGFzdGUoJ0FVUFI6ICcsIHNwcmludGYoJyUuM2YnLCByb3VuZChhdXByLCAzKSksIHNlcCA9ICcnKSkKYGBgCgoKYGBge3J9CmltYWdlX2xldmVsIDwtIHJlYWRfY3N2KCcuL291dC9wcm9iYWJpbGl0aWVzL3JldGNhbV9maWx0ZXJlZF8wLmNzdicsIGNvbF90eXBlcyA9IGNvbHMoKSkgJT4lCiAgICBtdXRhdGUobGFiZWwgPSBhcy5mYWN0b3IoaWZfZWxzZShzdHJfZGV0ZWN0KGltZ19sb2MsICdibGFjaycpLCAxLCAwKSksCiAgICAgICAgICAgaW1hZ2VfaWQgPSBhcy5udW1lcmljKGZpbGVfcGF0aF9zYW5zX2V4dChiYXNlbmFtZShpbWdfbG9jKSkpKSAlPiUKICAgIHNlbGVjdChpbWdfbG9jLCBpbWFnZV9pZCwgbGFiZWwsIHJldGNhbSA9IHByb2JhYmlsaXR5KSAlPiUKICAgIGJpbmRfY29scyhzZWxlY3QocmVhZF9jc3YoJy4vb3V0L3Byb2JhYmlsaXRpZXMvc2VnbWVudGF0aW9uc19iaW5hcml6ZWRfdGVzdC5jc3YnLCBjb2xfdHlwZXMgPSBjb2xzKCkpLCBmaWx0ZXJfMF9iaW5hcml6ZWQgPSBwcm9iYWJpbGl0eSkpCmBgYAoKYGBge3J9CnByZWQgPC0gcHJlZGljdGlvbihpbWFnZV9sZXZlbCRmaWx0ZXJfMF9iaW5hcml6ZWQsIGltYWdlX2xldmVsJGxhYmVsKQoKbnVtX3llcyA8LSBsZW5ndGgoaW1hZ2VfbGV2ZWwkbGFiZWxbaW1hZ2VfbGV2ZWwkbGFiZWwgPT0gMV0pCm51bV9ubyA8LSBsZW5ndGgoaW1hZ2VfbGV2ZWwkbGFiZWxbaW1hZ2VfbGV2ZWwkbGFiZWwgPT0gMF0pCgpwZXJmIDwtIHBlcmZvcm1hbmNlKHByZWQsJ3ByZWMnLCAncmVjJykKcGxvdChwZXJmLCBjb2xvcml6ZT1UUlVFLCBtYWluID0gJ01hbnVhbCBTZWdtZW50YXRpb25zOiBTa2VsZXRvbml6ZWQgLSAwIFRocmVzaG9sZCBNb2RlbCcsIHlsaW0gPSBjKDAsMSkpCmFibGluZShhID0gbnVtX3llcyAvIG51bV9ubywgYiA9IDAsIGNvbCA9ICdyZWQnLCBsdHkgPSAyKQphdXByIDwtIHBlcmZvcm1hbmNlKHByZWQsIG1lYXN1cmUgPSAnYXVjcHInKQphdXByIDwtIGF1cHJAeS52YWx1ZXNbWzFdXQp0ZXh0KDAuOCwgMC45LCBwYXN0ZSgnQVVQUjogJywgc3ByaW50ZignJS4zZicsIHJvdW5kKGF1cHIsIDMpKSwgc2VwID0gJycpKQpgYGA=